Note: This is test shard 4 of 8.
[==========] Running 3 tests from 3 test suites.
[----------] Global test environment set-up.
[----------] 1 test from AllTypesItest/1, where TypeParam = kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> >
[ RUN ] AllTypesItest/1.TestTimestampPadding
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20260810 22:56:19.997756 6570 test_util.cc:268] Using random seed: 1127539617
2026-08-10T22:56:20Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2026-08-10T22:56:20Z Disabled control of system clock
I20260810 22:56:20.026031 6570 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskC98JjP/build/debug/bin/kudu
/tmp/dist-test-taskC98JjP/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.6.106.190:36779
--webserver_interface=127.6.106.190
--webserver_port=0
--builtin_ntp_servers=127.6.106.148:34353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.6.106.190:36779 with env {}
W20260810 22:56:20.157166 6578 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:56:20.157486 6578 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:56:20.157521 6578 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:56:20.161742 6578 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
W20260810 22:56:20.161818 6578 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:56:20.161839 6578 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
W20260810 22:56:20.161857 6578 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
I20260810 22:56:20.167130 6578 master_runner.cc:386] Master server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.6.106.148:34353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal
--ipki_ca_key_size=768
--master_addresses=127.6.106.190:36779
--ipki_server_key_size=768
--openssl_security_level_override=0
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.6.106.190:36779
--rpc_reuseport=true
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb
--webserver_interface=127.6.106.190
--webserver_port=0
--never_fsync=true
--heap_profile_path=/tmp/kudu.6578
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/logs
--logbuflevel=-1
--logtostderr=true
Master server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 22:18:12 UTC on ccycloud.quasar-kzngpw.root.comops.site
build id 4579240
I20260810 22:56:20.168535 6578 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:56:20.170456 6578 file_cache.cc:493] Constructed file cache file cache with capacity 419430
I20260810 22:56:20.183835 6578 server_base.cc:1034] running on GCE node
W20260810 22:56:20.183889 6583 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.183929 6584 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.184449 6587 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:56:20.185211 6578 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:56:20.186991 6578 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:56:20.188242 6578 hybrid_clock.cc:648] HybridClock initialized: now 1786402580188217 us; error 60 us; skew 500 ppm
I20260810 22:56:20.195840 6578 webserver.cc:460] Webserver started at http://127.6.106.190:38509/ using document root <none> and password file <none>
I20260810 22:56:20.196885 6578 fs_manager.cc:359] Metadata directory not provided
I20260810 22:56:20.197010 6578 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:56:20.197438 6578 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:56:20.200245 6578 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/instance:
uuid: "d9ee52ac203e4e6daf3c5bc191701f39"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.201005 6578 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal/instance:
uuid: "d9ee52ac203e4e6daf3c5bc191701f39"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.206203 6578 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
I20260810 22:56:20.210108 6593 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.211901 6578 fs_manager.cc:725] Time spent opening block manager: real 0.004s user 0.003s sys 0.001s
I20260810 22:56:20.212155 6578 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data,/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal
uuid: "d9ee52ac203e4e6daf3c5bc191701f39"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.212324 6578 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal
metadata directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal
1 data directories: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:56:20.235253 6578 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:56:20.236214 6578 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:56:20.236500 6578 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:56:20.246166 6578 rpc_server.cc:307] RPC server started. Bound to: 127.6.106.190:36779
I20260810 22:56:20.246227 6651 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.106.190:36779 every 8 connection(s)
I20260810 22:56:20.247530 6578 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb
I20260810 22:56:20.252957 6652 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:20.254951 6570 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskC98JjP/build/debug/bin/kudu as pid 6578
I20260810 22:56:20.255175 6570 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/wal/instance
I20260810 22:56:20.261159 6652 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: Bootstrap starting.
I20260810 22:56:20.265465 6652 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:20.266659 6652 log.cc:824] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: Log is configured to *not* fsync() on all Append() calls
I20260810 22:56:20.269801 6652 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: No bootstrap required, opened a new log
I20260810 22:56:20.274362 6652 raft_consensus.cc:348] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } }
I20260810 22:56:20.274744 6652 raft_consensus.cc:374] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:20.274822 6652 raft_consensus.cc:729] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d9ee52ac203e4e6daf3c5bc191701f39, State: Initialized, Role: FOLLOWER
I20260810 22:56:20.275576 6652 consensus_queue.cc:260] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } }
I20260810 22:56:20.275799 6652 raft_consensus.cc:388] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Only one voter in the Raft config. Triggering election immediately
I20260810 22:56:20.275969 6652 raft_consensus.cc:482] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Starting leader election (initial election of a single-replica configuration)
I20260810 22:56:20.276134 6652 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:20.277164 6652 raft_consensus.cc:504] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } }
I20260810 22:56:20.277859 6652 leader_election.cc:302] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 1 responses out of 1 voters: 1 yes votes; 0 no votes. yes voters: d9ee52ac203e4e6daf3c5bc191701f39; no voters:
I20260810 22:56:20.278311 6652 leader_election.cc:288] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [CANDIDATE]: Term 1 election: Requested vote from peers
I20260810 22:56:20.278446 6657 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:56:20.278668 6657 raft_consensus.cc:686] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [term 1 LEADER]: Becoming Leader. State: Replica: d9ee52ac203e4e6daf3c5bc191701f39, State: Running, Role: LEADER
I20260810 22:56:20.279188 6657 consensus_queue.cc:237] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 1, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } }
I20260810 22:56:20.280153 6652 sys_catalog.cc:564] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [sys.catalog]: configured and running, proceeding with master startup.
I20260810 22:56:20.281731 6658 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 leader_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } } }
I20260810 22:56:20.281917 6658 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [sys.catalog]: This master's current role is: LEADER
I20260810 22:56:20.282380 6659 sys_catalog.cc:455] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d9ee52ac203e4e6daf3c5bc191701f39. Latest consensus state: current_term: 1 leader_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" committed_config { opid_index: -1 OBSOLETE_local: true peers { permanent_uuid: "d9ee52ac203e4e6daf3c5bc191701f39" member_type: VOTER last_known_addr { host: "127.6.106.190" port: 36779 } } }
I20260810 22:56:20.282476 6659 sys_catalog.cc:458] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39 [sys.catalog]: This master's current role is: LEADER
I20260810 22:56:20.282574 6667 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
I20260810 22:56:20.287360 6667 catalog_manager.cc:1484] Initializing Kudu cluster ID...
I20260810 22:56:20.294886 6667 catalog_manager.cc:1347] Generated new cluster ID: c9461b9ca63d487eafeafd1c5f0da1a5
I20260810 22:56:20.294983 6667 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
I20260810 22:56:20.308729 6667 catalog_manager.cc:1370] Generated new certificate authority record
I20260810 22:56:20.310096 6667 catalog_manager.cc:1504] Loading token signing keys...
I20260810 22:56:20.317987 6667 catalog_manager.cc:5868] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: Generated new TSK 0
I20260810 22:56:20.319036 6667 catalog_manager.cc:1514] Initializing in-progress tserver states...
I20260810 22:56:20.326264 6570 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskC98JjP/build/debug/bin/kudu
/tmp/dist-test-taskC98JjP/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.6.106.129:0
--local_ip_for_outbound_sockets=127.6.106.129
--webserver_interface=127.6.106.129
--webserver_port=0
--tserver_master_addrs=127.6.106.190:36779
--builtin_ntp_servers=127.6.106.148:34353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
W20260810 22:56:20.493139 6679 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:56:20.493440 6679 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:56:20.493487 6679 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:56:20.497875 6679 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:56:20.497960 6679 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:56:20.497977 6679 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:56:20.498015 6679 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.106.129
I20260810 22:56:20.502959 6679 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.6.106.148:34353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.6.106.129:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb
--webserver_interface=127.6.106.129
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.6.106.190:36779
--never_fsync=true
--heap_profile_path=/tmp/kudu.6679
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.6.106.129
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 22:18:12 UTC on ccycloud.quasar-kzngpw.root.comops.site
build id 4579240
I20260810 22:56:20.504113 6679 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:56:20.505900 6679 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:56:20.518482 6685 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.518504 6689 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.518539 6684 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:56:20.518893 6679 server_base.cc:1034] running on GCE node
I20260810 22:56:20.519356 6679 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:56:20.520102 6679 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:56:20.521301 6679 hybrid_clock.cc:648] HybridClock initialized: now 1786402580521280 us; error 34 us; skew 500 ppm
I20260810 22:56:20.524087 6679 webserver.cc:460] Webserver started at http://127.6.106.129:40417/ using document root <none> and password file <none>
I20260810 22:56:20.524732 6679 fs_manager.cc:359] Metadata directory not provided
I20260810 22:56:20.524832 6679 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:56:20.525094 6679 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:56:20.527498 6679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/instance:
uuid: "d8411147a1174ff8aeadf0e21a8edfc5"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.528172 6679 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal/instance:
uuid: "d8411147a1174ff8aeadf0e21a8edfc5"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.532398 6679 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.001s
I20260810 22:56:20.535262 6696 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.536711 6679 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20260810 22:56:20.536870 6679 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data,/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal
uuid: "d8411147a1174ff8aeadf0e21a8edfc5"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.537014 6679 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal
metadata directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal
1 data directories: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:56:20.570200 6679 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:56:20.571194 6679 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:56:20.571448 6679 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:56:20.572329 6679 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:56:20.573711 6679 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:56:20.573791 6679 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.573877 6679 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:56:20.573931 6679 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.593926 6679 rpc_server.cc:307] RPC server started. Bound to: 127.6.106.129:37745
I20260810 22:56:20.594018 6815 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.106.129:37745 every 8 connection(s)
I20260810 22:56:20.595073 6679 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb
I20260810 22:56:20.597193 6570 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskC98JjP/build/debug/bin/kudu as pid 6679
I20260810 22:56:20.597285 6570 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/wal/instance
I20260810 22:56:20.599954 6570 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskC98JjP/build/debug/bin/kudu
/tmp/dist-test-taskC98JjP/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.6.106.130:0
--local_ip_for_outbound_sockets=127.6.106.130
--webserver_interface=127.6.106.130
--webserver_port=0
--tserver_master_addrs=127.6.106.190:36779
--builtin_ntp_servers=127.6.106.148:34353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
I20260810 22:56:20.608711 6816 heartbeater.cc:347] Connected to a master server at 127.6.106.190:36779
I20260810 22:56:20.609027 6816 heartbeater.cc:464] Registering TS with master...
I20260810 22:56:20.609730 6816 heartbeater.cc:511] Master 127.6.106.190:36779 requested a full tablet report, sending...
I20260810 22:56:20.611855 6612 ts_manager.cc:194] Registered new tserver with Master: d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745)
I20260810 22:56:20.613332 6612 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.106.129:49085
W20260810 22:56:20.752743 6820 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:56:20.753105 6820 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:56:20.753193 6820 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:56:20.757486 6820 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:56:20.757609 6820 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:56:20.757663 6820 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:56:20.757788 6820 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.106.130
I20260810 22:56:20.762900 6820 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.6.106.148:34353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.6.106.130:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb
--webserver_interface=127.6.106.130
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.6.106.190:36779
--never_fsync=true
--heap_profile_path=/tmp/kudu.6820
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.6.106.130
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 22:18:12 UTC on ccycloud.quasar-kzngpw.root.comops.site
build id 4579240
I20260810 22:56:20.764178 6820 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:56:20.765828 6820 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:56:20.777848 6825 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.777781 6826 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:20.777935 6828 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:56:20.778162 6820 server_base.cc:1034] running on GCE node
I20260810 22:56:20.778841 6820 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:56:20.779666 6820 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:56:20.780869 6820 hybrid_clock.cc:648] HybridClock initialized: now 1786402580780833 us; error 42 us; skew 500 ppm
I20260810 22:56:20.783970 6820 webserver.cc:460] Webserver started at http://127.6.106.130:37083/ using document root <none> and password file <none>
I20260810 22:56:20.784713 6820 fs_manager.cc:359] Metadata directory not provided
I20260810 22:56:20.784817 6820 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:56:20.785070 6820 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:56:20.787446 6820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/instance:
uuid: "131b3fbf5d0048889a033c7aad167c31"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.788174 6820 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal/instance:
uuid: "131b3fbf5d0048889a033c7aad167c31"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.792411 6820 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.005s sys 0.000s
I20260810 22:56:20.795140 6834 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.796518 6820 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.001s sys 0.000s
I20260810 22:56:20.796700 6820 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data,/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal
uuid: "131b3fbf5d0048889a033c7aad167c31"
format_stamp: "Formatted at 2026-08-10 22:56:20 on dist-test-slave-p3t5"
I20260810 22:56:20.796833 6820 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal
metadata directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal
1 data directories: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:56:20.816138 6820 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:56:20.817075 6820 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:56:20.817351 6820 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:56:20.818265 6820 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:56:20.819396 6820 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:56:20.819446 6820 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.819521 6820 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:56:20.819561 6820 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:20.835189 6820 rpc_server.cc:307] RPC server started. Bound to: 127.6.106.130:45969
I20260810 22:56:20.835227 6957 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.106.130:45969 every 8 connection(s)
I20260810 22:56:20.836274 6820 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb
I20260810 22:56:20.840694 6570 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskC98JjP/build/debug/bin/kudu as pid 6820
I20260810 22:56:20.840790 6570 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/wal/instance
I20260810 22:56:20.843266 6570 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskC98JjP/build/debug/bin/kudu
/tmp/dist-test-taskC98JjP/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.6.106.131:0
--local_ip_for_outbound_sockets=127.6.106.131
--webserver_interface=127.6.106.131
--webserver_port=0
--tserver_master_addrs=127.6.106.190:36779
--builtin_ntp_servers=127.6.106.148:34353
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--flush_threshold_mb=1
--tablet_delta_store_major_compact_min_ratio=0.001
--consensus_max_batch_size_bytes=2097152 with env {}
I20260810 22:56:20.849761 6958 heartbeater.cc:347] Connected to a master server at 127.6.106.190:36779
I20260810 22:56:20.850064 6958 heartbeater.cc:464] Registering TS with master...
I20260810 22:56:20.850683 6958 heartbeater.cc:511] Master 127.6.106.190:36779 requested a full tablet report, sending...
I20260810 22:56:20.851925 6612 ts_manager.cc:194] Registered new tserver with Master: 131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969)
I20260810 22:56:20.852906 6612 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.106.130:60743
W20260810 22:56:20.997119 6962 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
W20260810 22:56:20.997437 6962 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
W20260810 22:56:20.997483 6962 flags.cc:425] Enabled unsafe flag: --never_fsync=true
W20260810 22:56:21.001995 6962 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
W20260810 22:56:21.002084 6962 flags.cc:425] Enabled experimental flag: --tablet_delta_store_major_compact_min_ratio=0.001
W20260810 22:56:21.002101 6962 flags.cc:425] Enabled experimental flag: --flush_threshold_mb=1
W20260810 22:56:21.002157 6962 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.6.106.131
I20260810 22:56:21.007177 6962 tablet_server_runner.cc:78] Tablet server non-default flags:
--builtin_ntp_poll_interval_ms=100
--builtin_ntp_servers=127.6.106.148:34353
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--consensus_max_batch_size_bytes=2097152
--fs_data_dirs=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data
--fs_wal_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal
--ipki_server_key_size=768
--openssl_security_level_override=0
--rpc_bind_addresses=127.6.106.131:0
--rpc_server_allow_ephemeral_ports=true
--metrics_log_interval_ms=1000
--server_dump_info_format=pb
--server_dump_info_path=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/info.pb
--webserver_interface=127.6.106.131
--webserver_port=0
--tablet_delta_store_major_compact_min_ratio=0.001
--flush_threshold_mb=1
--tserver_master_addrs=127.6.106.190:36779
--never_fsync=true
--heap_profile_path=/tmp/kudu.6962
--redact=none
--unlock_experimental_flags=true
--unlock_unsafe_flags=true
--enable_minidumps=false
--local_ip_for_outbound_sockets=127.6.106.131
--log_dir=/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/logs
--logbuflevel=-1
--logtostderr=true
Tablet server version:
kudu 1.18.0-SNAPSHOT
revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
build type DEBUG
built by systest at 10 Aug 2026 22:18:12 UTC on ccycloud.quasar-kzngpw.root.comops.site
build id 4579240
I20260810 22:56:21.008293 6962 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
I20260810 22:56:21.009848 6962 file_cache.cc:493] Constructed file cache file cache with capacity 419430
W20260810 22:56:21.020514 6971 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:56:21.020659 6962 server_base.cc:1034] running on GCE node
W20260810 22:56:21.020512 6969 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
W20260810 22:56:21.020512 6967 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
I20260810 22:56:21.021294 6962 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
I20260810 22:56:21.022051 6962 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
I20260810 22:56:21.023224 6962 hybrid_clock.cc:648] HybridClock initialized: now 1786402581023184 us; error 52 us; skew 500 ppm
I20260810 22:56:21.025271 6962 webserver.cc:460] Webserver started at http://127.6.106.131:34813/ using document root <none> and password file <none>
I20260810 22:56:21.025966 6962 fs_manager.cc:359] Metadata directory not provided
I20260810 22:56:21.026036 6962 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
I20260810 22:56:21.026292 6962 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
I20260810 22:56:21.028331 6962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/instance:
uuid: "48d765d5719f4b159d095ecafc56cea3"
format_stamp: "Formatted at 2026-08-10 22:56:21 on dist-test-slave-p3t5"
I20260810 22:56:21.028954 6962 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal/instance:
uuid: "48d765d5719f4b159d095ecafc56cea3"
format_stamp: "Formatted at 2026-08-10 22:56:21 on dist-test-slave-p3t5"
I20260810 22:56:21.033290 6962 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.002s
I20260810 22:56:21.036010 6977 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:21.037534 6962 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.000s sys 0.002s
I20260810 22:56:21.037740 6962 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data,/tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal
uuid: "48d765d5719f4b159d095ecafc56cea3"
format_stamp: "Formatted at 2026-08-10 22:56:21 on dist-test-slave-p3t5"
I20260810 22:56:21.037878 6962 fs_report.cc:383] FS layout report
--------------------
wal directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal
metadata directory: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal
1 data directories: /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/data
Total live blocks: 0
Total live bytes: 0
Total live bytes (after alignment): 0
Total number of LBM containers: 0 (0 full)
Did not check for missing blocks
Did not check for orphaned blocks
Total full LBM containers with extra space: 0 (0 repaired)
Total full LBM container extra space in bytes: 0 (0 repaired)
Total incomplete LBM containers: 0 (0 repaired)
Total LBM partial records: 0 (0 repaired)
Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
I20260810 22:56:21.079892 6962 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
I20260810 22:56:21.080861 6962 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
I20260810 22:56:21.081097 6962 kserver.cc:163] Server-wide thread pool size limit: 3276
I20260810 22:56:21.082015 6962 txn_system_client.cc:433] TxnSystemClient initialization is disabled...
I20260810 22:56:21.083186 6962 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
I20260810 22:56:21.083259 6962 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:21.083340 6962 ts_tablet_manager.cc:610] Registered 0 tablets
I20260810 22:56:21.083371 6962 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
I20260810 22:56:21.093833 6962 rpc_server.cc:307] RPC server started. Bound to: 127.6.106.131:36609
I20260810 22:56:21.093899 7096 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.6.106.131:36609 every 8 connection(s)
I20260810 22:56:21.094887 6962 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/info.pb
I20260810 22:56:21.104110 6570 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskC98JjP/build/debug/bin/kudu as pid 6962
I20260810 22:56:21.104218 6570 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/wal/instance
I20260810 22:56:21.107154 7097 heartbeater.cc:347] Connected to a master server at 127.6.106.190:36779
I20260810 22:56:21.107472 7097 heartbeater.cc:464] Registering TS with master...
I20260810 22:56:21.108098 7097 heartbeater.cc:511] Master 127.6.106.190:36779 requested a full tablet report, sending...
I20260810 22:56:21.109292 6612 ts_manager.cc:194] Registered new tserver with Master: 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.110005 6612 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.6.106.131:58761
I20260810 22:56:21.119747 6570 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
I20260810 22:56:21.136601 6612 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:39536:
name: "all-types-table"
schema {
columns {
name: "key"
type: INT16
is_key: true
is_nullable: false
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int8_val"
type: INT8
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int16_val"
type: INT16
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int32_val"
type: INT32
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "int64_val"
type: INT64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "timestamp_val"
type: UNIXTIME_MICROS
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "date_val"
type: DATE
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "string_val"
type: STRING
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "varchar_val"
type: VARCHAR
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
length: 65535
}
immutable: false
}
columns {
name: "bool_val"
type: BOOL
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "float_val"
type: FLOAT
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "double_val"
type: DOUBLE
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "binary_val"
type: BINARY
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
immutable: false
}
columns {
name: "decimal32_val"
type: DECIMAL32
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 9
scale: 0
}
immutable: false
}
columns {
name: "decimal64_val"
type: DECIMAL64
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 18
scale: 0
}
immutable: false
}
columns {
name: "decimal128_val"
type: DECIMAL128
is_key: false
is_nullable: true
encoding: AUTO_ENCODING
compression: DEFAULT_COMPRESSION
cfile_block_size: 0
type_attributes {
precision: 38
scale: 0
}
immutable: false
}
}
num_replicas: 3
split_rows_range_bounds {
rows: "<redacted>""\004\001\000\000\000\252*\004\001\000\000\000TU"
indirect_data: "<redacted>"""
}
partition_schema {
range_schema {
columns {
name: "key"
}
}
}
W20260810 22:56:21.140538 6612 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table all-types-table in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
I20260810 22:56:21.162971 6746 tablet_service.cc:1467] Processing CreateTablet for tablet 4fc5146ddfff440594fd98d716c8bf1c (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:56:21.164532 6746 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fc5146ddfff440594fd98d716c8bf1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.162969 7024 tablet_service.cc:1467] Processing CreateTablet for tablet eb23104048554e98ab96128e229998cc (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:56:21.162801 6745 tablet_service.cc:1467] Processing CreateTablet for tablet d4f81674db9541b88bfb97fefcc225ff (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:56:21.166296 6745 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4f81674db9541b88bfb97fefcc225ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.162801 6744 tablet_service.cc:1467] Processing CreateTablet for tablet eb23104048554e98ab96128e229998cc (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:56:21.166647 7024 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb23104048554e98ab96128e229998cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.166913 6744 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb23104048554e98ab96128e229998cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.166363 6884 tablet_service.cc:1467] Processing CreateTablet for tablet eb23104048554e98ab96128e229998cc (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 21844 <= VALUES
I20260810 22:56:21.166536 6885 tablet_service.cc:1467] Processing CreateTablet for tablet d4f81674db9541b88bfb97fefcc225ff (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:56:21.162801 7026 tablet_service.cc:1467] Processing CreateTablet for tablet 4fc5146ddfff440594fd98d716c8bf1c (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:56:21.168084 6884 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet eb23104048554e98ab96128e229998cc. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.168193 7026 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fc5146ddfff440594fd98d716c8bf1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.162803 7025 tablet_service.cc:1467] Processing CreateTablet for tablet d4f81674db9541b88bfb97fefcc225ff (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION 10922 <= VALUES < 21844
I20260810 22:56:21.168941 7025 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4f81674db9541b88bfb97fefcc225ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.173960 6885 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d4f81674db9541b88bfb97fefcc225ff. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.175024 7118 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: Bootstrap starting.
I20260810 22:56:21.177194 7119 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: Bootstrap starting.
I20260810 22:56:21.166363 6886 tablet_service.cc:1467] Processing CreateTablet for tablet 4fc5146ddfff440594fd98d716c8bf1c (DEFAULT_TABLE table=all-types-table [id=de7a3cbf69c548ceb4f5afda239abd97]), partition=RANGE (key) PARTITION VALUES < 10922
I20260810 22:56:21.177650 7118 tablet_bootstrap.cc:654] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.177776 6886 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 4fc5146ddfff440594fd98d716c8bf1c. 1 dirs total, 0 dirs full, 0 dirs failed
I20260810 22:56:21.178558 7117 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Bootstrap starting.
I20260810 22:56:21.179472 7118 log.cc:824] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: Log is configured to *not* fsync() on all Append() calls
I20260810 22:56:21.180336 7119 tablet_bootstrap.cc:654] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.181123 7117 tablet_bootstrap.cc:654] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.181412 7119 log.cc:824] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: Log is configured to *not* fsync() on all Append() calls
I20260810 22:56:21.182215 7118 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: No bootstrap required, opened a new log
I20260810 22:56:21.182435 7118 ts_tablet_manager.cc:1397] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: Time spent bootstrapping tablet: real 0.008s user 0.006s sys 0.000s
I20260810 22:56:21.182922 7117 log.cc:824] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Log is configured to *not* fsync() on all Append() calls
I20260810 22:56:21.184167 7119 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: No bootstrap required, opened a new log
I20260810 22:56:21.184419 7119 ts_tablet_manager.cc:1397] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: Time spent bootstrapping tablet: real 0.007s user 0.005s sys 0.000s
I20260810 22:56:21.185505 7117 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: No bootstrap required, opened a new log
I20260810 22:56:21.185734 7117 ts_tablet_manager.cc:1397] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Time spent bootstrapping tablet: real 0.007s user 0.006s sys 0.000s
I20260810 22:56:21.186553 7118 raft_consensus.cc:348] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.186908 7118 raft_consensus.cc:374] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.186972 7118 raft_consensus.cc:729] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48d765d5719f4b159d095ecafc56cea3, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.187467 7118 consensus_queue.cc:260] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.187697 7119 raft_consensus.cc:348] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.187995 7119 raft_consensus.cc:374] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.188069 7119 raft_consensus.cc:729] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 131b3fbf5d0048889a033c7aad167c31, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.188500 7097 heartbeater.cc:503] Master 127.6.106.190:36779 was elected leader, sending a full tablet report...
I20260810 22:56:21.188596 7118 ts_tablet_manager.cc:1428] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3: Time spent starting tablet: real 0.006s user 0.006s sys 0.000s
I20260810 22:56:21.188537 7119 consensus_queue.cc:260] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.188927 7118 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: Bootstrap starting.
I20260810 22:56:21.189581 7119 ts_tablet_manager.cc:1428] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31: Time spent starting tablet: real 0.005s user 0.006s sys 0.000s
I20260810 22:56:21.189965 6958 heartbeater.cc:503] Master 127.6.106.190:36779 was elected leader, sending a full tablet report...
I20260810 22:56:21.190169 7118 tablet_bootstrap.cc:654] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.190819 7117 raft_consensus.cc:348] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.191103 7119 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31: Bootstrap starting.
I20260810 22:56:21.191110 7117 raft_consensus.cc:374] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.191166 7117 raft_consensus.cc:729] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d8411147a1174ff8aeadf0e21a8edfc5, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.191610 7117 consensus_queue.cc:260] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.192435 7119 tablet_bootstrap.cc:654] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.192477 6816 heartbeater.cc:503] Master 127.6.106.190:36779 was elected leader, sending a full tablet report...
I20260810 22:56:21.192826 7117 ts_tablet_manager.cc:1428] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Time spent starting tablet: real 0.007s user 0.005s sys 0.000s
I20260810 22:56:21.193274 7117 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5: Bootstrap starting.
I20260810 22:56:21.194677 7117 tablet_bootstrap.cc:654] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.195072 7119 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31: No bootstrap required, opened a new log
I20260810 22:56:21.195137 7119 ts_tablet_manager.cc:1397] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20260810 22:56:21.195626 7119 raft_consensus.cc:348] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.195713 7119 raft_consensus.cc:374] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.195745 7119 raft_consensus.cc:729] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 131b3fbf5d0048889a033c7aad167c31, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.195885 7119 consensus_queue.cc:260] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.196161 7119 ts_tablet_manager.cc:1428] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20260810 22:56:21.196293 7119 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: Bootstrap starting.
I20260810 22:56:21.197743 7119 tablet_bootstrap.cc:654] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.198872 7118 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: No bootstrap required, opened a new log
I20260810 22:56:21.198982 7118 ts_tablet_manager.cc:1397] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: Time spent bootstrapping tablet: real 0.010s user 0.003s sys 0.000s
I20260810 22:56:21.199235 7117 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5: No bootstrap required, opened a new log
I20260810 22:56:21.199298 7117 ts_tablet_manager.cc:1397] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5: Time spent bootstrapping tablet: real 0.006s user 0.002s sys 0.000s
I20260810 22:56:21.199381 7119 tablet_bootstrap.cc:492] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: No bootstrap required, opened a new log
I20260810 22:56:21.199457 7119 ts_tablet_manager.cc:1397] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: Time spent bootstrapping tablet: real 0.003s user 0.001s sys 0.001s
I20260810 22:56:21.199714 7118 raft_consensus.cc:348] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.199848 7118 raft_consensus.cc:374] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.199747 7117 raft_consensus.cc:348] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.199888 7118 raft_consensus.cc:729] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48d765d5719f4b159d095ecafc56cea3, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.199913 7117 raft_consensus.cc:374] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.199954 7117 raft_consensus.cc:729] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d8411147a1174ff8aeadf0e21a8edfc5, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.199887 7119 raft_consensus.cc:348] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.200006 7119 raft_consensus.cc:374] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.200043 7119 raft_consensus.cc:729] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 131b3fbf5d0048889a033c7aad167c31, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.200034 7118 consensus_queue.cc:260] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.200089 7117 consensus_queue.cc:260] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.200160 7119 consensus_queue.cc:260] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.200333 7118 ts_tablet_manager.cc:1428] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20260810 22:56:21.200382 7117 ts_tablet_manager.cc:1428] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
I20260810 22:56:21.200438 7119 ts_tablet_manager.cc:1428] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: Time spent starting tablet: real 0.001s user 0.000s sys 0.001s
I20260810 22:56:21.200539 7118 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3: Bootstrap starting.
I20260810 22:56:21.200554 7117 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5: Bootstrap starting.
I20260810 22:56:21.201958 7117 tablet_bootstrap.cc:654] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.202862 7118 tablet_bootstrap.cc:654] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3: Neither blocks nor log segments found. Creating new log.
I20260810 22:56:21.204340 7117 tablet_bootstrap.cc:492] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5: No bootstrap required, opened a new log
I20260810 22:56:21.204423 7117 ts_tablet_manager.cc:1397] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.002s
I20260810 22:56:21.204658 7118 tablet_bootstrap.cc:492] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3: No bootstrap required, opened a new log
I20260810 22:56:21.204813 7118 ts_tablet_manager.cc:1397] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3: Time spent bootstrapping tablet: real 0.004s user 0.002s sys 0.000s
I20260810 22:56:21.205224 7118 raft_consensus.cc:348] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.205233 7117 raft_consensus.cc:348] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.205343 7118 raft_consensus.cc:374] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.205340 7117 raft_consensus.cc:374] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
I20260810 22:56:21.205389 7118 raft_consensus.cc:729] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 48d765d5719f4b159d095ecafc56cea3, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.205395 7117 raft_consensus.cc:729] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d8411147a1174ff8aeadf0e21a8edfc5, State: Initialized, Role: FOLLOWER
I20260810 22:56:21.205530 7117 consensus_queue.cc:260] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.205526 7118 consensus_queue.cc:260] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.205830 7118 ts_tablet_manager.cc:1428] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3: Time spent starting tablet: real 0.001s user 0.001s sys 0.000s
I20260810 22:56:21.205850 7117 ts_tablet_manager.cc:1428] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5: Time spent starting tablet: real 0.001s user 0.000s sys 0.002s
I20260810 22:56:21.234601 7124 raft_consensus.cc:482] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:56:21.234820 7124 raft_consensus.cc:504] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.235970 7124 leader_election.cc:288] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.240500 6767 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4f81674db9541b88bfb97fefcc225ff" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" is_pre_election: true
I20260810 22:56:21.240743 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4f81674db9541b88bfb97fefcc225ff" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3" is_pre_election: true
I20260810 22:56:21.240999 6767 raft_consensus.cc:2454] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 0.
I20260810 22:56:21.241057 7047 raft_consensus.cc:2454] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 0.
I20260810 22:56:21.241698 6837 leader_election.cc:302] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b3fbf5d0048889a033c7aad167c31, 48d765d5719f4b159d095ecafc56cea3; no voters:
I20260810 22:56:21.242043 7124 raft_consensus.cc:2781] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:56:21.242129 7124 raft_consensus.cc:482] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:56:21.242190 7124 raft_consensus.cc:3037] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.243296 7124 raft_consensus.cc:504] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.243669 7124 leader_election.cc:288] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 election: Requested vote from peers d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.244055 6767 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4f81674db9541b88bfb97fefcc225ff" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8411147a1174ff8aeadf0e21a8edfc5"
I20260810 22:56:21.244179 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d4f81674db9541b88bfb97fefcc225ff" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3"
I20260810 22:56:21.244217 6767 raft_consensus.cc:3037] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.244355 7047 raft_consensus.cc:3037] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.245335 6767 raft_consensus.cc:2454] T d4f81674db9541b88bfb97fefcc225ff P d8411147a1174ff8aeadf0e21a8edfc5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 1.
I20260810 22:56:21.245362 7047 raft_consensus.cc:2454] T d4f81674db9541b88bfb97fefcc225ff P 48d765d5719f4b159d095ecafc56cea3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 1.
I20260810 22:56:21.245680 6837 leader_election.cc:302] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b3fbf5d0048889a033c7aad167c31, 48d765d5719f4b159d095ecafc56cea3; no voters:
I20260810 22:56:21.245923 7124 raft_consensus.cc:2781] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:56:21.246201 7124 raft_consensus.cc:686] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [term 1 LEADER]: Becoming Leader. State: Replica: 131b3fbf5d0048889a033c7aad167c31, State: Running, Role: LEADER
I20260810 22:56:21.246624 7124 consensus_queue.cc:237] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.250141 6609 catalog_manager.cc:5495] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 reported cstate change: term changed from 0 to 1, leader changed from <none> to 131b3fbf5d0048889a033c7aad167c31 (127.6.106.130). New cstate: current_term: 1 leader_uuid: "131b3fbf5d0048889a033c7aad167c31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } health_report { overall_health: UNKNOWN } } }
W20260810 22:56:21.337733 6959 tablet.cc:2369] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20260810 22:56:21.345985 7098 tablet.cc:2369] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: Can't schedule compaction. Clean time has not been advanced past its initial value.
W20260810 22:56:21.346721 6818 tablet.cc:2369] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5: Can't schedule compaction. Clean time has not been advanced past its initial value.
I20260810 22:56:21.372411 7125 raft_consensus.cc:482] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:56:21.372619 7125 raft_consensus.cc:504] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.374049 7125 leader_election.cc:288] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.378809 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fc5146ddfff440594fd98d716c8bf1c" candidate_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3" is_pre_election: true
I20260810 22:56:21.379051 7047 raft_consensus.cc:2454] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d8411147a1174ff8aeadf0e21a8edfc5 in term 0.
I20260810 22:56:21.378963 6908 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fc5146ddfff440594fd98d716c8bf1c" candidate_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "131b3fbf5d0048889a033c7aad167c31" is_pre_election: true
I20260810 22:56:21.379267 6908 raft_consensus.cc:2454] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d8411147a1174ff8aeadf0e21a8edfc5 in term 0.
I20260810 22:56:21.379513 6700 leader_election.cc:302] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 48d765d5719f4b159d095ecafc56cea3, d8411147a1174ff8aeadf0e21a8edfc5; no voters:
I20260810 22:56:21.379906 7125 raft_consensus.cc:2781] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:56:21.380005 7125 raft_consensus.cc:482] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:56:21.380085 7125 raft_consensus.cc:3037] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.381211 7125 raft_consensus.cc:504] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.381722 7125 leader_election.cc:288] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [CANDIDATE]: Term 1 election: Requested vote from peers 131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.382133 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fc5146ddfff440594fd98d716c8bf1c" candidate_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3"
I20260810 22:56:21.382166 6908 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "4fc5146ddfff440594fd98d716c8bf1c" candidate_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "131b3fbf5d0048889a033c7aad167c31"
I20260810 22:56:21.382282 7047 raft_consensus.cc:3037] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.382304 6908 raft_consensus.cc:3037] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.383225 7047 raft_consensus.cc:2454] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d8411147a1174ff8aeadf0e21a8edfc5 in term 1.
I20260810 22:56:21.383515 6908 raft_consensus.cc:2454] T 4fc5146ddfff440594fd98d716c8bf1c P 131b3fbf5d0048889a033c7aad167c31 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d8411147a1174ff8aeadf0e21a8edfc5 in term 1.
I20260810 22:56:21.383562 6700 leader_election.cc:302] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 48d765d5719f4b159d095ecafc56cea3, d8411147a1174ff8aeadf0e21a8edfc5; no voters:
I20260810 22:56:21.383741 7125 raft_consensus.cc:2781] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:56:21.384025 7125 raft_consensus.cc:686] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [term 1 LEADER]: Becoming Leader. State: Replica: d8411147a1174ff8aeadf0e21a8edfc5, State: Running, Role: LEADER
I20260810 22:56:21.384611 7125 consensus_queue.cc:237] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.388365 6609 catalog_manager.cc:5495] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 reported cstate change: term changed from 0 to 1, leader changed from <none> to d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129). New cstate: current_term: 1 leader_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } health_report { overall_health: UNKNOWN } } }
I20260810 22:56:21.412439 7124 raft_consensus.cc:482] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
I20260810 22:56:21.412608 7124 raft_consensus.cc:504] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.413120 7124 leader_election.cc:288] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.413820 6767 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb23104048554e98ab96128e229998cc" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" is_pre_election: true
I20260810 22:56:21.413825 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb23104048554e98ab96128e229998cc" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3" is_pre_election: true
I20260810 22:56:21.414008 7047 raft_consensus.cc:2454] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 0.
I20260810 22:56:21.414008 6767 raft_consensus.cc:2454] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 0.
I20260810 22:56:21.414350 6837 leader_election.cc:302] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b3fbf5d0048889a033c7aad167c31, d8411147a1174ff8aeadf0e21a8edfc5; no voters:
I20260810 22:56:21.414543 7124 raft_consensus.cc:2781] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Leader pre-election won for term 1
I20260810 22:56:21.414614 7124 raft_consensus.cc:482] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
I20260810 22:56:21.414651 7124 raft_consensus.cc:3037] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.415613 7124 raft_consensus.cc:504] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.415997 7124 leader_election.cc:288] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 election: Requested vote from peers d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745), 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609)
I20260810 22:56:21.416417 6767 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb23104048554e98ab96128e229998cc" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d8411147a1174ff8aeadf0e21a8edfc5"
I20260810 22:56:21.416440 7047 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "eb23104048554e98ab96128e229998cc" candidate_uuid: "131b3fbf5d0048889a033c7aad167c31" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "48d765d5719f4b159d095ecafc56cea3"
I20260810 22:56:21.416572 6767 raft_consensus.cc:3037] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.416594 7047 raft_consensus.cc:3037] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 0 FOLLOWER]: Advancing to term 1
I20260810 22:56:21.417572 7047 raft_consensus.cc:2454] T eb23104048554e98ab96128e229998cc P 48d765d5719f4b159d095ecafc56cea3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 1.
I20260810 22:56:21.417747 6767 raft_consensus.cc:2454] T eb23104048554e98ab96128e229998cc P d8411147a1174ff8aeadf0e21a8edfc5 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 131b3fbf5d0048889a033c7aad167c31 in term 1.
I20260810 22:56:21.417960 6837 leader_election.cc:302] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 131b3fbf5d0048889a033c7aad167c31, 48d765d5719f4b159d095ecafc56cea3; no voters:
I20260810 22:56:21.418161 7124 raft_consensus.cc:2781] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 1 FOLLOWER]: Leader election won for term 1
I20260810 22:56:21.418277 7124 raft_consensus.cc:686] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [term 1 LEADER]: Becoming Leader. State: Replica: 131b3fbf5d0048889a033c7aad167c31, State: Running, Role: LEADER
I20260810 22:56:21.418437 7124 consensus_queue.cc:237] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } }
I20260810 22:56:21.420287 6609 catalog_manager.cc:5495] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 reported cstate change: term changed from 0 to 1, leader changed from <none> to 131b3fbf5d0048889a033c7aad167c31 (127.6.106.130). New cstate: current_term: 1 leader_uuid: "131b3fbf5d0048889a033c7aad167c31" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 } health_report { overall_health: UNKNOWN } } }
I20260810 22:56:21.735056 7124 consensus_queue.cc:1035] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:21.741478 7134 consensus_queue.cc:1035] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:21.831612 7134 consensus_queue.cc:1035] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d8411147a1174ff8aeadf0e21a8edfc5" member_type: VOTER last_known_addr { host: "127.6.106.129" port: 37745 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:21.833200 7134 consensus_queue.cc:1035] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:21.962734 7125 consensus_queue.cc:1035] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "131b3fbf5d0048889a033c7aad167c31" member_type: VOTER last_known_addr { host: "127.6.106.130" port: 45969 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:21.967672 7153 consensus_queue.cc:1035] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [LEADER]: Connected to new peer: Peer: permanent_uuid: "48d765d5719f4b159d095ecafc56cea3" member_type: VOTER last_known_addr { host: "127.6.106.131" port: 36609 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
I20260810 22:56:23.091735 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling FlushMRSOp(d4f81674db9541b88bfb97fefcc225ff): perf score=1.000000
I20260810 22:56:23.099992 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling FlushMRSOp(4fc5146ddfff440594fd98d716c8bf1c): perf score=1.000000
W20260810 22:56:23.335130 7104 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.6.106.129:37745, user_credentials={real_user=slave}} blocked reactor thread for 96345.3us
I20260810 22:56:23.350108 7098 maintenance_manager.cc:419] P 48d765d5719f4b159d095ecafc56cea3: Scheduling FlushMRSOp(4fc5146ddfff440594fd98d716c8bf1c): perf score=1.000000
W20260810 22:56:23.492880 7103 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.6.106.130:45969, user_credentials={real_user=slave}} blocked reactor thread for 88621.2us
W20260810 22:56:23.883653 6982 dir_manager.cc:174] Insufficient disk space under path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/data: will retry after 10 seconds: IO error: Insufficient disk space to allocate 0 bytes under path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-2/data/data (504156160 bytes available vs 518351011 bytes reserved) (error 28)
W20260810 22:56:23.904271 6982 tablet_replica_mm_ops.cc:214] T 4fc5146ddfff440594fd98d716c8bf1c P 48d765d5719f4b159d095ecafc56cea3: failed to flush MRS: IO error: Failed to open DiskRowSet for flush: tablet 4fc5146ddfff440594fd98d716c8bf1c: unable to open output file for column double_val DOUBLE NULLABLE: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 4fc5146ddfff440594fd98d716c8bf1c's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
F20260810 22:56:23.904587 6982 tablet_replica_mm_ops.cc:215] Check failed: tablet->HasBeenStopped() Unrecoverable flush failure caused by error: IO error: Failed to open DiskRowSet for flush: tablet 4fc5146ddfff440594fd98d716c8bf1c: unable to open output file for column double_val DOUBLE NULLABLE: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 4fc5146ddfff440594fd98d716c8bf1c's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
*** Check failure stack trace: ***
@ 0x7f9432f639ad google::LogMessage::Fail() at ??:0
@ 0x7f9432f6688a google::LogMessage::SendToLog() at ??:0
@ 0x7f9432f634de google::LogMessage::Flush() at ??:0
@ 0x7f9432f64a19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f943bcdc299 kudu::tablet::FlushMRSOp::Perform() at ??:0
@ 0x7f9435119674 kudu::MaintenanceManager::LaunchOp() at ??:0
@ 0x7f943511673c _ZZN4kudu18MaintenanceManager18RunSchedulerThreadEvENKUlvE_clEv at ??:0
@ 0x7f943511a88c _ZNSt17_Function_handlerIFvvEZN4kudu18MaintenanceManager18RunSchedulerThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f94351b53d7 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f94351b5c83 _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f94351b73f2 _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f94351a6512 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f94367ea6db start_thread at ??:0
@ 0x7f9431a6161f clone at ??:0
W20260810 22:56:24.303681 7103 outbound_call.cc:322] RPC callback for RPC call kudu.tserver.TabletServerService.Write -> {remote=127.6.106.130:45969, user_credentials={real_user=slave}} blocked reactor thread for 49500.1us
I20260810 22:56:24.394428 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:24.406366 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20260810 22:56:24.407761 6700 connection.cc:535] client connection to 127.6.106.131:36609 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:56:24.407953 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:56:24.408560 6837 connection.cc:535] client connection to 127.6.106.131:36609 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:56:24.408704 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20260810 22:56:24.415304 7238 negotiation.cc:337] Failed RPC negotiation. Trace:
0810 22:56:24.378383 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.6.106.131:36609 (local address 127.0.0.1:41790)
0810 22:56:24.378731 (+ 348us) negotiation.cc:107] Waiting for socket to connect
0810 22:56:24.378749 (+ 18us) client_negotiation.cc:174] Beginning negotiation
0810 22:56:24.378848 (+ 99us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
0810 22:56:24.413830 (+ 34982us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
Metrics: {"client-negotiator.queue_time_us":262,"thread_start_us":139,"threads_started":1}
W20260810 22:56:24.423280 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20260810 22:56:24.423758 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
W20260810 22:56:24.423856 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:24.440626 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:24.543823 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: FlushMRSOp(d4f81674db9541b88bfb97fefcc225ff) complete. Timing: real 1.451s user 0.636s sys 0.078s Metrics: {"bytes_written":471972,"cfile_init":1,"compiler_manager_pool.queue_time_us":10218,"compiler_manager_pool.run_cpu_time_us":306318,"compiler_manager_pool.run_wall_time_us":831034,"dirs.queue_time_us":5058,"dirs.run_cpu_time_us":217,"dirs.run_wall_time_us":43932,"drs_written":1,"lbm_read_time_us":88,"lbm_reads_lt_1ms":4,"lbm_write_time_us":19799,"lbm_writes_10-100_ms":1,"lbm_writes_lt_1ms":85,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":297,"threads_started":3,"wal-append.queue_time_us":210}
I20260810 22:56:24.545370 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling FlushMRSOp(4fc5146ddfff440594fd98d716c8bf1c): perf score=1.937278
I20260810 22:56:24.546662 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: FlushMRSOp(4fc5146ddfff440594fd98d716c8bf1c) complete. Timing: real 1.446s user 0.637s sys 0.041s Metrics: {"bytes_written":436310,"cfile_init":1,"compiler_manager_pool.queue_time_us":10309,"compiler_manager_pool.run_cpu_time_us":299520,"compiler_manager_pool.run_wall_time_us":660834,"dirs.queue_time_us":251,"dirs.run_cpu_time_us":295,"dirs.run_wall_time_us":74444,"drs_written":1,"lbm_read_time_us":59,"lbm_reads_lt_1ms":4,"lbm_write_time_us":3052,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"spinlock_wait_cycles":226688,"thread_start_us":326,"threads_started":3,"wal-append.queue_time_us":259}
I20260810 22:56:24.548499 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling FlushMRSOp(d4f81674db9541b88bfb97fefcc225ff): perf score=1.937278
I20260810 22:56:24.654426 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:24.656389 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:24.695772 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:24.987576 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:24.989401 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: FlushMRSOp(4fc5146ddfff440594fd98d716c8bf1c) complete. Timing: real 0.444s user 0.381s sys 0.010s Metrics: {"bytes_written":436310,"cfile_init":1,"dirs.queue_time_us":61,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2502,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":152,"threads_started":1,"wal-append.queue_time_us":270}
I20260810 22:56:24.990774 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:24.994354 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling FlushMRSOp(eb23104048554e98ab96128e229998cc): perf score=1.937278
I20260810 22:56:25.016296 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: FlushMRSOp(d4f81674db9541b88bfb97fefcc225ff) complete. Timing: real 0.468s user 0.434s sys 0.016s Metrics: {"bytes_written":471972,"cfile_init":1,"dirs.queue_time_us":68,"dirs.run_cpu_time_us":364,"dirs.run_wall_time_us":39679,"drs_written":1,"lbm_read_time_us":61,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2992,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":130,"threads_started":1,"wal-append.queue_time_us":229}
I20260810 22:56:25.017024 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling FlushMRSOp(eb23104048554e98ab96128e229998cc): perf score=1.937278
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:25.039718 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:25.367899 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: FlushMRSOp(eb23104048554e98ab96128e229998cc) complete. Timing: real 0.351s user 0.348s sys 0.000s Metrics: {"bytes_written":472920,"cfile_init":1,"dirs.queue_time_us":65,"dirs.run_cpu_time_us":178,"dirs.run_wall_time_us":1407,"drs_written":1,"lbm_read_time_us":44,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2366,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":130,"threads_started":1,"wal-append.queue_time_us":279}
I20260810 22:56:25.368554 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling UndoDeltaBlockGCOp(eb23104048554e98ab96128e229998cc): 63434 bytes on disk
I20260810 22:56:25.369294 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: UndoDeltaBlockGCOp(eb23104048554e98ab96128e229998cc) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.369239 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: FlushMRSOp(eb23104048554e98ab96128e229998cc) complete. Timing: real 0.375s user 0.368s sys 0.001s Metrics: {"bytes_written":472920,"cfile_init":1,"dirs.queue_time_us":65,"dirs.run_cpu_time_us":276,"dirs.run_wall_time_us":1160,"drs_written":1,"lbm_read_time_us":34,"lbm_reads_lt_1ms":4,"lbm_write_time_us":2777,"lbm_writes_lt_1ms":86,"peak_mem_usage":0,"rows_written":10000,"thread_start_us":103,"threads_started":1,"wal-append.queue_time_us":205}
I20260810 22:56:25.369807 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling UndoDeltaBlockGCOp(4fc5146ddfff440594fd98d716c8bf1c): 62923 bytes on disk
I20260810 22:56:25.370007 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling UndoDeltaBlockGCOp(eb23104048554e98ab96128e229998cc): 63434 bytes on disk
I20260810 22:56:25.370216 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: UndoDeltaBlockGCOp(4fc5146ddfff440594fd98d716c8bf1c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":54,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.370589 6818 maintenance_manager.cc:419] P d8411147a1174ff8aeadf0e21a8edfc5: Scheduling UndoDeltaBlockGCOp(d4f81674db9541b88bfb97fefcc225ff): 62923 bytes on disk
I20260810 22:56:25.370746 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: UndoDeltaBlockGCOp(eb23104048554e98ab96128e229998cc) complete. Timing: real 0.001s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":66,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.370939 6702 maintenance_manager.cc:643] P d8411147a1174ff8aeadf0e21a8edfc5: UndoDeltaBlockGCOp(d4f81674db9541b88bfb97fefcc225ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":47,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.371213 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling UndoDeltaBlockGCOp(4fc5146ddfff440594fd98d716c8bf1c): 62923 bytes on disk
I20260810 22:56:25.371562 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: UndoDeltaBlockGCOp(4fc5146ddfff440594fd98d716c8bf1c) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":46,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.371918 6959 maintenance_manager.cc:419] P 131b3fbf5d0048889a033c7aad167c31: Scheduling UndoDeltaBlockGCOp(d4f81674db9541b88bfb97fefcc225ff): 62923 bytes on disk
I20260810 22:56:25.372220 6839 maintenance_manager.cc:643] P 131b3fbf5d0048889a033c7aad167c31: UndoDeltaBlockGCOp(d4f81674db9541b88bfb97fefcc225ff) complete. Timing: real 0.000s user 0.000s sys 0.000s Metrics: {"cfile_init":1,"lbm_read_time_us":38,"lbm_reads_lt_1ms":4}
I20260810 22:56:25.437764 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:25.437778 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:25.463932 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:26.028270 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:26.028563 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:26.047889 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:26.805238 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20260810 22:56:26.865100 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:26.865115 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:26.886626 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:26.889019 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
W20260810 22:56:27.089649 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
I20260810 22:56:27.944967 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:27.945036 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:27.968236 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:29.026180 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:29.026180 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:29.047258 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:29.208671 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20260810 22:56:29.636215 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:56:29.636612 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:56:29.637271 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
W20260810 22:56:29.637416 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
I20260810 22:56:30.107511 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:30.107595 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:30.129920 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:31.191262 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:31.192894 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:31.218098 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:31.839944 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20260810 22:56:31.898694 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
W20260810 22:56:32.093351 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
I20260810 22:56:32.274971 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:32.274971 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:32.301198 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:33.354756 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:33.355644 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:33.379035 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:34.092001 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20260810 22:56:34.311594 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
I20260810 22:56:34.434828 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:34.435487 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:34.456106 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:34.742599 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:56:34.743400 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
W20260810 22:56:34.808077 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:56:35.513355 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:35.513928 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:35.537134 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:36.521040 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20260810 22:56:36.606194 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:36.606999 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:36.633165 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:36.816603 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
W20260810 22:56:36.990326 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 26: this message will repeat every 5th retry.
I20260810 22:56:37.689014 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:37.689035 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:37.716408 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:38.877082 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:38.878048 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:38.940996 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:39.048595 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20260810 22:56:39.115355 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20260810 22:56:39.431677 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 31: this message will repeat every 5th retry.
W20260810 22:56:39.863327 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:56:40.005523 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:40.006095 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:40.027501 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:40.057287 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:56:41.083956 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:41.083936 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:41.106540 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:41.360743 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20260810 22:56:41.449431 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
W20260810 22:56:41.789312 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 36: this message will repeat every 5th retry.
I20260810 22:56:42.160676 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:42.160751 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:42.183459 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:43.237979 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:43.238026 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:43.260507 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:43.859691 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20260810 22:56:44.032330 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
W20260810 22:56:44.059448 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 41: this message will repeat every 5th retry.
I20260810 22:56:44.316025 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:44.316491 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:44.338488 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:44.878077 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:56:45.316701 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:56:45.395188 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:45.395519 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:45.414914 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:46.154486 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
W20260810 22:56:46.472203 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20260810 22:56:46.472301 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:46.472548 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:46.497681 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:46.549074 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 46: this message will repeat every 5th retry.
I20260810 22:56:47.558772 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:47.558772 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:47.584132 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:48.515429 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20260810 22:56:48.643694 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:56:48.644062 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:48.664255 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:48.945688 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
W20260810 22:56:49.072615 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 51: this message will repeat every 5th retry.
I20260810 22:56:49.719341 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:49.719947 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:49.738891 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:50.002641 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:56:50.538121 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:56:50.798806 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:50.799384 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:50.822746 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:50.842973 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20260810 22:56:51.644083 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
W20260810 22:56:51.702530 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 56: this message will repeat every 5th retry.
I20260810 22:56:51.892855 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:51.893000 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:51.925875 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:52.986364 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:52.986656 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:53.008180 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:53.379841 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20260810 22:56:54.064385 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:54.064385 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:54.084362 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:54.201699 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
W20260810 22:56:54.320403 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 61: this message will repeat every 5th retry.
I20260810 22:56:55.139487 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:55.140333 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:55.161504 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:55.302960 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:56:55.847460 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20260810 22:56:56.037719 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:56:56.218250 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:56.218506 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:56.241376 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:56.996546 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
W20260810 22:56:57.018929 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 66: this message will repeat every 5th retry.
I20260810 22:56:57.297942 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:57.298130 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:57.319468 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:58.378994 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
I20260810 22:56:58.387442 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:58.387588 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:58.410861 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:56:59.468029 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:56:59.468339 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:56:59.494010 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:56:59.569726 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20260810 22:56:59.652179 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 71: this message will repeat every 5th retry.
W20260810 22:57:00.511605 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:57:00.512377 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20260810 22:57:00.551707 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:00.552024 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:00.574545 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:01.201023 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:57:01.635798 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:01.637204 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:01.661515 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:02.301316 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
W20260810 22:57:02.305401 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 76: this message will repeat every 5th retry.
I20260810 22:57:02.719851 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:02.720361 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:02.739830 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:02.955651 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20260810 22:57:03.801893 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:03.801927 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:03.822453 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:04.845352 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
I20260810 22:57:04.883297 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:04.883337 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20260810 22:57:04.895499 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 81: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:04.910362 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:05.471480 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20260810 22:57:05.947537 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:57:05.971760 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:05.971715 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:05.992933 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:06.681746 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:57:07.052454 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:07.052454 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:07.076046 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:07.144398 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20260810 22:57:07.255750 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 86: this message will repeat every 5th retry.
W20260810 22:57:07.938150 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20260810 22:57:08.132426 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:08.133033 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:08.152734 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:09.216888 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:09.217250 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:09.241284 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:09.530663 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
W20260810 22:57:09.804399 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 91: this message will repeat every 5th retry.
I20260810 22:57:10.298327 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:10.298704 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:10.323638 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:10.512744 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20260810 22:57:10.953706 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:57:11.383044 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:11.383324 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:11.403899 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:12.031783 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:57:12.032691 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
W20260810 22:57:12.063452 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 96: this message will repeat every 5th retry.
I20260810 22:57:12.461412 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:12.461952 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:12.482805 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:12.726732 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
I20260810 22:57:13.541473 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:13.542482 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:13.569875 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:14.633811 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:14.633744 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:14.657235 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:14.715978 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20260810 22:57:14.748528 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 101: this message will repeat every 5th retry.
W20260810 22:57:15.130699 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
I20260810 22:57:15.717226 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:15.717257 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:15.747520 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:16.077203 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:57:16.809763 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:16.810132 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:16.839076 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:17.259297 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20260810 22:57:17.308879 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:57:17.309914 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 106: this message will repeat every 5th retry.
W20260810 22:57:17.532678 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20260810 22:57:17.900600 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:17.901068 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:17.922777 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:18.983136 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:18.983500 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:19.006368 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:19.654634 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
W20260810 22:57:19.687474 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20260810 22:57:19.878541 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 111: this message will repeat every 5th retry.
I20260810 22:57:20.071027 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:20.071043 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:20.092362 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:21.150030 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:21.150724 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:21.172586 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:21.338253 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:57:22.182371 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20260810 22:57:22.221764 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20260810 22:57:22.243431 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:22.244637 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:22.272769 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:22.457644 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 116: this message will repeat every 5th retry.
W20260810 22:57:22.613981 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:57:23.329305 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:23.329303 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:23.350493 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:24.407933 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:24.408164 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
W20260810 22:57:24.413291 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:24.436105 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:24.849787 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20260810 22:57:24.897387 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 121: this message will repeat every 5th retry.
I20260810 22:57:25.498636 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:25.498852 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:25.517834 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:26.392429 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
I20260810 22:57:26.578450 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:26.578658 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:26.599911 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:27.019028 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
W20260810 22:57:27.234524 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20260810 22:57:27.421461 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 126: this message will repeat every 5th retry.
I20260810 22:57:27.656785 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:27.656944 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:27.682328 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:28.038318 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:57:28.738445 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:28.739010 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:28.763657 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:29.498071 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
W20260810 22:57:29.517529 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20260810 22:57:29.822731 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:29.822891 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:29.849362 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:30.083827 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 131: this message will repeat every 5th retry.
I20260810 22:57:30.915237 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:30.915715 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:30.939738 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:31.576901 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:57:31.999212 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:31.999256 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:32.020666 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:32.133737 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
W20260810 22:57:32.186084 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20260810 22:57:32.509608 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 136: this message will repeat every 5th retry.
I20260810 22:57:33.080339 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:33.081210 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:33.106858 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:33.284423 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:57:34.165015 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:34.165048 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:34.188681 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:34.841563 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20260810 22:57:34.870653 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
W20260810 22:57:35.092708 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 141: this message will repeat every 5th retry.
I20260810 22:57:35.244772 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:35.245064 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:35.272090 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:36.335319 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:36.335319 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:36.360188 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:36.619190 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:57:37.079181 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20260810 22:57:37.416692 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:37.417167 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:37.437773 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:37.672461 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20260810 22:57:37.770258 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 146: this message will repeat every 5th retry.
W20260810 22:57:38.334308 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 8 similar messages]
I20260810 22:57:38.493848 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:38.494019 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:38.515991 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:39.545579 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
I20260810 22:57:39.573510 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:39.574280 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:39.597108 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:39.905539 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
W20260810 22:57:40.507929 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 151: this message will repeat every 5th retry.
I20260810 22:57:40.654462 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:40.654912 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:40.675423 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:41.642794 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:57:41.736466 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:41.737155 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:41.757236 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:42.163601 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20260810 22:57:42.414613 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
I20260810 22:57:42.814209 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:42.814914 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:42.837568 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:42.859273 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 156: this message will repeat every 5th retry.
W20260810 22:57:43.398097 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:57:43.898201 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:43.898231 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:43.923610 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:44.840879 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20260810 22:57:44.980288 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:44.980623 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:45.002947 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:45.059228 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
W20260810 22:57:45.495482 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 161: this message will repeat every 5th retry.
I20260810 22:57:46.061514 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:46.063287 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:46.088269 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:46.823045 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
I20260810 22:57:47.155249 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:47.155264 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:47.175429 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:47.357470 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
W20260810 22:57:47.825451 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
W20260810 22:57:48.146739 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 166: this message will repeat every 5th retry.
I20260810 22:57:48.235189 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:48.235394 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:48.253871 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:48.611598 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:57:49.309612 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:49.309734 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:49.329012 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:49.713264 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20260810 22:57:50.387435 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:50.387456 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20260810 22:57:50.406739 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:50.412575 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:50.540158 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 171: this message will repeat every 5th retry.
I20260810 22:57:51.469823 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:51.470309 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:51.489446 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:52.134505 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:57:52.135672 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20260810 22:57:52.559444 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:52.561575 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:52.587505 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:52.798229 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
W20260810 22:57:53.097384 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 176: this message will repeat every 5th retry.
I20260810 22:57:53.644330 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:53.644384 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:53.665266 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:53.822512 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:57:54.609983 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
I20260810 22:57:54.722908 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:54.722929 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:54.749235 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:55.458431 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
I20260810 22:57:55.807286 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:55.807600 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
W20260810 22:57:55.823388 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 181: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:55.827823 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:57:56.885784 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:57:56.886868 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:56.909392 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:57.196148 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:57:57.200224 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20260810 22:57:57.972697 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:57.973022 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:57.996052 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:58.006544 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20260810 22:57:58.115507 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 186: this message will repeat every 5th retry.
W20260810 22:57:58.856536 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
I20260810 22:57:59.057197 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:57:59.058799 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:57:59.084296 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:57:59.752686 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20260810 22:58:00.140475 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:00.140540 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:00.163710 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:00.214596 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
W20260810 22:58:00.911446 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 191: this message will repeat every 5th retry.
I20260810 22:58:01.224100 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:01.226486 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:01.246206 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:02.287099 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:58:02.288040 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20260810 22:58:02.303516 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:02.303512 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:02.324981 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:02.448838 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
I20260810 22:58:03.387966 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:03.388078 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:03.413465 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:03.681931 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 196: this message will repeat every 5th retry.
W20260810 22:58:03.996924 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
I20260810 22:58:04.469079 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:04.469208 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:04.490917 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:04.902088 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20260810 22:58:05.040663 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20260810 22:58:05.550712 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:05.550730 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:05.571941 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:06.359980 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 201: this message will repeat every 5th retry.
I20260810 22:58:06.629285 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:06.629339 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:06.650378 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:07.348667 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 22:58:07.396739 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20260810 22:58:07.419945 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20260810 22:58:07.709496 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:07.709781 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:07.731828 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:58:08.798077 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:08.798461 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:08.819825 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:08.839565 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 206: this message will repeat every 5th retry.
W20260810 22:58:09.404330 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:58:09.764722 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20260810 22:58:09.837958 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20260810 22:58:09.878815 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:09.879101 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:09.902115 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:58:10.959725 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:10.960215 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:10.982028 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:11.629441 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 211: this message will repeat every 5th retry.
I20260810 22:58:12.038535 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:12.038625 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:12.064885 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:12.168107 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20260810 22:58:12.198621 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20260810 22:58:12.809434 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
I20260810 22:58:13.119216 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:13.119259 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:13.141899 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:58:14.200832 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:14.200850 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:14.220929 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:14.311278 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 216: this message will repeat every 5th retry.
W20260810 22:58:14.582199 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:58:14.583113 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
W20260810 22:58:14.944336 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20260810 22:58:15.277901 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:15.279464 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:15.302250 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
I20260810 22:58:16.355422 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
I20260810 22:58:16.356045 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:16.379645 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:16.930307 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
W20260810 22:58:17.014014 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 221: this message will repeat every 5th retry.
I20260810 22:58:17.435258 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:17.435539 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:17.459216 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:17.505226 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20260810 22:58:17.839949 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
I20260810 22:58:18.516196 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:18.516301 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.129:37745, tserver@127.6.106.130:45969
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:18.543125 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:19.229900 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20260810 22:58:19.245225 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 226: this message will repeat every 5th retry.
I20260810 22:58:19.606083 6745 tablet_service.cc:1429] Tablet server has 1 leaders and 0 scanners
I20260810 22:58:19.606077 6886 tablet_service.cc:1429] Tablet server has 2 leaders and 0 scanners
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
d9ee52ac203e4e6daf3c5bc191701f39 | 127.6.106.190:36779 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
----------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.6.106.148:34353 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
131b3fbf5d0048889a033c7aad167c31 | 127.6.106.130:45969 | HEALTHY | <none> | 2 | 0
d8411147a1174ff8aeadf0e21a8edfc5 | 127.6.106.129:37745 | HEALTHY | <none> | 1 | 0
48d765d5719f4b159d095ecafc56cea3 | 127.6.106.131:36609 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 3
Error from 127.6.106.131:36609: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
--------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------+------------------------------------------
flush_threshold_mb | 1 | runtime,experimental | 127.6.106.130:45969, 127.6.106.129:37745
ipki_server_key_size | 768 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.129 | experimental | 127.6.106.129:37745
local_ip_for_outbound_sockets | 127.6.106.130 | experimental | 127.6.106.130:45969
never_fsync | true | unsafe,advanced | 127.6.106.130:45969, 127.6.106.129:37745
openssl_security_level_override | 0 | unsafe,hidden | 127.6.106.130:45969, 127.6.106.129:37745
rpc_server_allow_ephemeral_ports | true | unsafe | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_format | pb | hidden | 127.6.106.130:45969, 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-0/data/info.pb | hidden | 127.6.106.129:37745
server_dump_info_path | /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/ts-1/data/info.pb | hidden | 127.6.106.130:45969
tablet_delta_store_major_compact_min_ratio | 0.001 | experimental | 127.6.106.130:45969, 127.6.106.129:37745
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.6.106.148:34353 | 127.6.106.130:45969, 127.6.106.129:37745
time_source | builtin | 127.6.106.130:45969, 127.6.106.129:37745
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.6.106.190:36779, tserver@127.6.106.130:45969, tserver@127.6.106.129:37745
Tablet Summary
Tablet 4fc5146ddfff440594fd98d716c8bf1c of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING [LEADER]
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | [config not available] | | |
Tablet d4f81674db9541b88bfb97fefcc225ff of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
Tablet eb23104048554e98ab96128e229998cc of table 'all-types-table' is under-replicated: 1 replica(s) not RUNNING
d8411147a1174ff8aeadf0e21a8edfc5 (127.6.106.129:37745): RUNNING
131b3fbf5d0048889a033c7aad167c31 (127.6.106.130:45969): RUNNING [LEADER]
48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): TS unavailable
All reported replicas are:
A = d8411147a1174ff8aeadf0e21a8edfc5
B = 131b3fbf5d0048889a033c7aad167c31
C = 48d765d5719f4b159d095ecafc56cea3
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
-----------------+----+------------------+---------------+---------+------------+------------------+-------------
all-types-table | 3 | UNDER_REPLICATED | 3 | 0 | 0 | 3 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 3
First Quartile | 3
Median | 3
Third Quartile | 3
Maximum | 3
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 3
Tables | 1
Tablets | 3
Replicas | 9
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 3 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 3 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 3 had errors
Corruption: table consistency check error: 1 out of 1 table(s) are not healthy
FAILED
I20260810 22:58:19.630614 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Runtime error: ksck discovered errors
W20260810 22:58:19.755703 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:58:19.781075 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
I20260810 22:58:20.424960 6601 maintenance_manager.cc:419] P d9ee52ac203e4e6daf3c5bc191701f39: Scheduling FlushMRSOp(00000000000000000000000000000000): perf score=0.033374
W20260810 22:58:20.427488 6599 dir_manager.cc:174] Insufficient disk space under path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/data: will retry after 10 seconds: IO error: Insufficient disk space to allocate 0 bytes under path /tmp/dist-test-taskC98JjP/test-tmp/all_types-itest.3.AllTypesItest_1.TestTimestampPadding.1786402579943502-6570-0/minicluster-data/master-0/data/data (499228672 bytes available vs 518351011 bytes reserved) (error 28)
W20260810 22:58:20.428069 6599 tablet_replica_mm_ops.cc:214] T 00000000000000000000000000000000 P d9ee52ac203e4e6daf3c5bc191701f39: failed to flush MRS: IO error: Failed to open DiskRowSet for flush: tablet 00000000000000000000000000000000: unable to open output file for column entry_type INT8 NOT NULL: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 00000000000000000000000000000000's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
F20260810 22:58:20.428174 6599 tablet_replica_mm_ops.cc:215] Check failed: tablet->HasBeenStopped() Unrecoverable flush failure caused by error: IO error: Failed to open DiskRowSet for flush: tablet 00000000000000000000000000000000: unable to open output file for column entry_type INT8 NOT NULL: No directories could be added (1 dirs total, 0 failed, 1 full): IO error: No directories available in 00000000000000000000000000000000's directory group (1 dirs total, 0 failed, 1 full) (error 28) (error 19)
*** Check failure stack trace: ***
@ 0x7f2c6d6d59ad google::LogMessage::Fail() at ??:0
@ 0x7f2c6d6d888a google::LogMessage::SendToLog() at ??:0
@ 0x7f2c6d6d54de google::LogMessage::Flush() at ??:0
@ 0x7f2c6d6d6a19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f2c7644e299 kudu::tablet::FlushMRSOp::Perform() at ??:0
@ 0x7f2c6f88b674 kudu::MaintenanceManager::LaunchOp() at ??:0
@ 0x7f2c6f88873c _ZZN4kudu18MaintenanceManager18RunSchedulerThreadEvENKUlvE_clEv at ??:0
@ 0x7f2c6f88c88c _ZNSt17_Function_handlerIFvvEZN4kudu18MaintenanceManager18RunSchedulerThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f2c6f9273d7 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f2c6f927c83 _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f2c6f9293f2 _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x61d838 std::function<>::operator()() at ??:0
@ 0x7f2c6f918512 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f2c70f5c6db start_thread at ??:0
@ 0x7f2c6c1d361f clone at ??:0
W20260810 22:58:21.013952 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 22:58:21.074559 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 22:58:21.861594 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 231: this message will repeat every 5th retry.
W20260810 22:58:21.927323 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20260810 22:58:22.570608 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:58:23.066910 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:58:24.149365 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 236: this message will repeat every 5th retry.
W20260810 22:58:24.543958 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:58:24.983776 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:58:25.045961 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:58:26.569298 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 241: this message will repeat every 5th retry.
W20260810 22:58:26.915100 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:58:27.851060 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:58:28.084587 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:58:28.884485 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 246: this message will repeat every 5th retry.
W20260810 22:58:29.520010 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:58:30.585314 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:58:30.618299 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:58:31.291314 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 251: this message will repeat every 5th retry.
W20260810 22:58:32.255313 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:58:32.837846 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:58:33.290503 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:58:33.966907 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 256: this message will repeat every 5th retry.
W20260810 22:58:34.714594 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:58:35.401999 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:58:35.901881 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:58:36.450315 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 261: this message will repeat every 5th retry.
W20260810 22:58:37.302796 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:58:37.842495 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:58:38.359325 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:58:39.111625 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 266: this message will repeat every 5th retry.
W20260810 22:58:39.745133 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:58:40.154927 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:58:41.143392 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:58:41.377388 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 271: this message will repeat every 5th retry.
W20260810 22:58:42.200084 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:58:42.466511 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:58:43.521294 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:58:43.993542 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 276: this message will repeat every 5th retry.
W20260810 22:58:44.661116 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:58:45.040640 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:58:46.233834 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:58:46.479044 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 281: this message will repeat every 5th retry.
W20260810 22:58:47.260073 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:58:47.453218 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:58:48.604888 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:58:49.090927 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 286: this message will repeat every 5th retry.
W20260810 22:58:49.742887 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:58:49.929207 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:58:51.331736 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:58:51.509055 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 291: this message will repeat every 5th retry.
W20260810 22:58:52.437340 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:58:52.500238 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:58:53.792860 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:58:53.793684 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 296: this message will repeat every 5th retry.
W20260810 22:58:54.903059 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:58:55.054855 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:58:56.155315 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 301: this message will repeat every 5th retry.
W20260810 22:58:56.398249 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:58:57.556993 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:58:57.959331 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:58:58.498323 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 306: this message will repeat every 5th retry.
W20260810 22:58:58.805209 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:59:00.143460 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:59:00.302582 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:59:01.037998 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 311: this message will repeat every 5th retry.
W20260810 22:59:01.568284 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:02.443142 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:59:02.546756 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:59:03.647730 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 316: this message will repeat every 5th retry.
W20260810 22:59:03.896641 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:04.779381 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:59:05.053295 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:59:06.055768 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 321: this message will repeat every 5th retry.
W20260810 22:59:06.606341 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:07.222128 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:59:07.746299 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:59:08.788093 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 326: this message will repeat every 5th retry.
W20260810 22:59:09.038763 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:09.440461 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:59:10.349500 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:59:11.161976 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 331: this message will repeat every 5th retry.
W20260810 22:59:11.663326 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:11.891988 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:59:12.539201 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:59:13.475890 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 336: this message will repeat every 5th retry.
W20260810 22:59:14.069077 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:14.354943 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:59:15.273010 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:59:16.115878 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 341: this message will repeat every 5th retry.
W20260810 22:59:16.749039 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20260810 22:59:16.835565 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:17.605489 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:59:18.263660 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 346: this message will repeat every 5th retry.
W20260810 22:59:19.029841 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:59:19.390228 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 22:59:20.107793 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
W20260810 22:59:20.683182 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 351: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 341) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 22:59:20.841709 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 341) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 22:59:21.085971 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (60 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 22:59:21.145465 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (60 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 22:59:21.461159 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:59:22.288511 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:59:22.913074 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:59:23.243286 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 356: this message will repeat every 5th retry.
W20260810 22:59:24.080662 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:59:24.493395 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:25.447827 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:59:25.591253 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 361: this message will repeat every 5th retry.
W20260810 22:59:26.914029 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:59:27.574363 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:27.901017 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:59:28.002461 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 366: this message will repeat every 5th retry.
W20260810 22:59:29.687669 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:59:29.688519 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:59:30.237304 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:59:30.746052 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 371: this message will repeat every 5th retry.
W20260810 22:59:32.200155 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:59:32.549243 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:59:33.028730 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:33.449862 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 376: this message will repeat every 5th retry.
W20260810 22:59:34.818348 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:34.819056 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:59:34.819751 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:59:35.792765 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 381: this message will repeat every 5th retry.
W20260810 22:59:37.491604 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:59:37.578060 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:59:38.193279 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:59:38.194079 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 386: this message will repeat every 5th retry.
W20260810 22:59:39.924660 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:59:39.925439 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:59:40.206406 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:59:40.752797 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 391: this message will repeat every 5th retry.
W20260810 22:59:42.531194 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:59:42.686872 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:59:43.472245 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:43.473171 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 396: this message will repeat every 5th retry.
W20260810 22:59:45.149276 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 22:59:45.150146 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:59:45.201350 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:59:46.028149 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 401: this message will repeat every 5th retry.
W20260810 22:59:47.618010 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:59:47.868476 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:59:48.465266 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 406: this message will repeat every 5th retry.
W20260810 22:59:48.953696 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:59:50.113283 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:59:50.353924 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:50.354724 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:59:50.746071 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 411: this message will repeat every 5th retry.
W20260810 22:59:52.356848 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:59:52.860131 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:59:53.252702 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 416: this message will repeat every 5th retry.
W20260810 22:59:54.221540 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 22:59:54.763466 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:59:55.373215 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 22:59:55.374130 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:59:55.918946 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 421: this message will repeat every 5th retry.
W20260810 22:59:57.288134 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:59:57.907406 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 22:59:58.552337 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 426: this message will repeat every 5th retry.
W20260810 22:59:59.439332 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 22:59:59.825431 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 23:00:00.288697 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 23:00:00.759205 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:00:00.896502 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 431: this message will repeat every 5th retry.
W20260810 23:00:02.424551 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 23:00:02.847265 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 23:00:03.294194 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 436: this message will repeat every 5th retry.
W20260810 23:00:04.445454 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:04.902850 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 23:00:05.210817 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 23:00:05.942322 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:00:06.035779 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 441: this message will repeat every 5th retry.
W20260810 23:00:07.342819 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 23:00:07.572327 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 23:00:08.340155 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 446: this message will repeat every 5th retry.
W20260810 23:00:09.390149 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 23:00:09.517701 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:10.500859 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 23:00:11.082883 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 451: this message will repeat every 5th retry.
W20260810 23:00:11.341302 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:00:11.863523 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 23:00:13.139534 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 23:00:13.557796 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 456: this message will repeat every 5th retry.
W20260810 23:00:14.605581 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:14.643239 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 23:00:15.830925 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 23:00:15.972170 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 461: this message will repeat every 5th retry.
W20260810 23:00:16.566406 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:00:17.144065 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 23:00:18.374891 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 23:00:18.525606 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 466: this message will repeat every 5th retry.
W20260810 23:00:19.417842 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 23:00:19.967207 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:00:21.068279 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 471: this message will repeat every 5th retry.
W20260810 23:00:21.133052 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20260810 23:00:21.158483 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (120 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:00:21.217303 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (120 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:00:21.712455 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:00:21.793457 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:00:22.069422 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:00:23.415838 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 476: this message will repeat every 5th retry.
W20260810 23:00:23.762313 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 23:00:24.234438 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 23:00:25.367676 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:00:25.900640 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 481: this message will repeat every 5th retry.
W20260810 23:00:26.216090 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 23:00:26.488816 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 23:00:26.866001 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:00:28.654737 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 486: this message will repeat every 5th retry.
W20260810 23:00:28.828657 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 23:00:28.901571 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 23:00:30.436035 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:30.967890 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 491: this message will repeat every 5th retry.
W20260810 23:00:31.305251 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 23:00:31.377538 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 23:00:31.971395 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:00:33.329959 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 496: this message will repeat every 5th retry.
W20260810 23:00:33.575423 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 23:00:33.922807 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 23:00:35.843451 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:00:35.844589 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 501: this message will repeat every 5th retry.
W20260810 23:00:36.040414 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 23:00:36.201200 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 23:00:37.029768 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:00:38.084375 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 23:00:38.373185 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 506: this message will repeat every 5th retry.
W20260810 23:00:38.525039 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 23:00:40.523382 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 23:00:40.750399 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 23:00:40.797276 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 511: this message will repeat every 5th retry.
W20260810 23:00:41.279573 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:00:42.334811 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:00:43.073910 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 23:00:43.270020 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 516: this message will repeat every 5th retry.
W20260810 23:00:43.286954 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 23:00:45.445508 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 23:00:45.682847 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 521: this message will repeat every 5th retry.
W20260810 23:00:46.014094 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 23:00:46.667937 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:00:47.393401 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:00:48.014333 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 23:00:48.298478 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 526: this message will repeat every 5th retry.
W20260810 23:00:48.384123 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 23:00:50.577610 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 23:00:50.661080 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 23:00:50.826428 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 531: this message will repeat every 5th retry.
W20260810 23:00:51.764448 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:52.730458 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:00:53.150604 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 23:00:53.254860 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 23:00:53.435582 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 536: this message will repeat every 5th retry.
W20260810 23:00:55.690649 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 23:00:55.937683 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 23:00:55.987625 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 541: this message will repeat every 5th retry.
W20260810 23:00:56.846618 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:00:57.734503 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:00:58.140874 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 546: this message will repeat every 5th retry.
W20260810 23:00:58.242615 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 23:00:58.644929 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 23:01:00.675932 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 551: this message will repeat every 5th retry.
W20260810 23:01:00.690769 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 23:01:01.177961 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 23:01:02.208552 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:01:02.806763 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:03.204298 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 23:01:03.290913 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 556: this message will repeat every 5th retry.
W20260810 23:01:03.822106 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 23:01:05.823328 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 561: this message will repeat every 5th retry.
W20260810 23:01:05.933813 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 23:01:06.390164 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 23:01:07.384297 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:01:07.932382 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:01:08.268625 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 566: this message will repeat every 5th retry.
W20260810 23:01:08.329449 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 23:01:08.999568 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 23:01:10.593493 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 571: this message will repeat every 5th retry.
W20260810 23:01:10.777328 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 23:01:11.523730 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 23:01:12.897759 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 11 similar messages]
W20260810 23:01:12.898746 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 576: this message will repeat every 5th retry.
W20260810 23:01:13.088045 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:01:13.229389 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 23:01:14.086499 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 23:01:15.689313 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 581: this message will repeat every 5th retry.
W20260810 23:01:15.693004 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 23:01:16.474424 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 23:01:18.153473 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:01:18.154311 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 586: this message will repeat every 5th retry.
W20260810 23:01:18.304003 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:01:18.304891 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 23:01:18.812987 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 23:01:20.870114 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 591: this message will repeat every 5th retry.
W20260810 23:01:21.135275 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20260810 23:01:21.232544 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (180 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:01:21.288328 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (180 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:01:21.399123 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:01:23.272010 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:01:23.503376 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:23.630944 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:01:23.631732 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 596: this message will repeat every 5th retry.
W20260810 23:01:23.780536 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20260810 23:01:23.993428 11012 consensus_queue.cc:579] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Leader has been unable to successfully communicate with peer 48d765d5719f4b159d095ecafc56cea3 for more than 300 seconds (300.250s)
W20260810 23:01:23.995093 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
I20260810 23:01:24.203568 11053 consensus_queue.cc:579] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 [LEADER]: Leader has been unable to successfully communicate with peer 48d765d5719f4b159d095ecafc56cea3 for more than 300 seconds (300.363s)
I20260810 23:01:24.359498 11100 consensus_queue.cc:579] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 [LEADER]: Leader has been unable to successfully communicate with peer 48d765d5719f4b159d095ecafc56cea3 for more than 300 seconds (300.563s)
W20260810 23:01:26.057554 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 601: this message will repeat every 5th retry.
W20260810 23:01:26.365242 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 23:01:26.671902 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 23:01:28.489275 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 606: this message will repeat every 5th retry.
W20260810 23:01:28.627151 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:28.628060 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 23:01:28.980250 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:01:29.258469 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 23:01:30.980037 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 23:01:31.203542 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 611: this message will repeat every 5th retry.
W20260810 23:01:31.736810 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 23:01:33.488927 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 23:01:33.669438 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:01:33.683313 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 616: this message will repeat every 5th retry.
W20260810 23:01:34.254779 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 23:01:34.256148 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:01:35.977483 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 23:01:36.177969 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 621: this message will repeat every 5th retry.
W20260810 23:01:36.829569 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 23:01:38.583448 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 23:01:38.762506 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:38.880280 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 626: this message will repeat every 5th retry.
W20260810 23:01:39.316450 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 23:01:39.396946 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:01:41.140340 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 631: this message will repeat every 5th retry.
W20260810 23:01:41.348045 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 23:01:41.732031 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 23:01:43.605947 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 636: this message will repeat every 5th retry.
W20260810 23:01:44.183727 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:44.184530 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 23:01:44.221016 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 23:01:44.815878 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:01:46.328797 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 641: this message will repeat every 5th retry.
W20260810 23:01:46.583504 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 23:01:46.605340 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 23:01:48.789505 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 646: this message will repeat every 5th retry.
W20260810 23:01:48.945490 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 23:01:48.988231 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 23:01:49.458297 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:01:50.129895 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:01:51.193285 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 651: this message will repeat every 5th retry.
W20260810 23:01:51.443434 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 23:01:51.765863 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 23:01:53.601189 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 656: this message will repeat every 5th retry.
W20260810 23:01:54.010097 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 23:01:54.655860 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:01:54.656685 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 23:01:55.720017 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:01:56.299572 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 661: this message will repeat every 5th retry.
W20260810 23:01:56.475314 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 23:01:57.013690 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 23:01:58.722532 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 666: this message will repeat every 5th retry.
W20260810 23:01:58.955381 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 23:01:59.429033 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 23:01:59.809890 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:00.996809 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:01.231199 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 23:02:01.513494 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 671: this message will repeat every 5th retry.
W20260810 23:02:01.709791 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 23:02:03.903254 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 23:02:04.038269 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 676: this message will repeat every 5th retry.
W20260810 23:02:04.267164 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 23:02:04.832913 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:06.180404 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 23:02:06.227854 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:06.228693 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 681: this message will repeat every 5th retry.
W20260810 23:02:06.538735 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 23:02:08.721154 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 686: this message will repeat every 5th retry.
W20260810 23:02:08.899689 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 23:02:08.952075 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 23:02:09.970232 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:11.049171 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 691: this message will repeat every 5th retry.
W20260810 23:02:11.284155 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 23:02:11.439363 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:11.582836 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 23:02:13.467347 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 696: this message will repeat every 5th retry.
W20260810 23:02:13.720717 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 23:02:14.193802 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 23:02:15.036768 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:02:15.531769 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 701: this message will repeat every 5th retry.
W20260810 23:02:15.983599 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 23:02:16.737000 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:16.794884 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 23:02:18.178993 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 706: this message will repeat every 5th retry.
W20260810 23:02:18.691023 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 23:02:19.308568 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 23:02:20.279040 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:20.481184 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 711: this message will repeat every 5th retry.
W20260810 23:02:20.911188 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 23:02:21.272768 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (240 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:02:21.427996 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (241 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:02:21.825086 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 23:02:21.980360 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:23.036669 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 716: this message will repeat every 5th retry.
W20260810 23:02:23.417521 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
W20260810 23:02:24.110926 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:02:24.472412 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:02:25.311806 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:25.696178 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 721: this message will repeat every 5th retry.
W20260810 23:02:25.891564 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 23:02:26.846904 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 23:02:27.276392 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:28.096930 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 23:02:28.242213 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 726: this message will repeat every 5th retry.
W20260810 23:02:29.152935 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 23:02:30.608755 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:30.609551 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 23:02:30.711102 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 731: this message will repeat every 5th retry.
W20260810 23:02:31.617390 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 23:02:32.528869 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:33.057816 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 736: this message will repeat every 5th retry.
W20260810 23:02:33.062296 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 23:02:33.978230 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 23:02:35.615535 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:35.616470 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 23:02:35.635789 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 741: this message will repeat every 5th retry.
W20260810 23:02:36.081274 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 23:02:37.734395 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:38.175637 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 746: this message will repeat every 5th retry.
W20260810 23:02:38.303881 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 23:02:38.459004 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 23:02:40.622169 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 751: this message will repeat every 5th retry.
W20260810 23:02:40.775218 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:40.776049 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 23:02:41.155017 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 23:02:43.055228 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:02:43.056248 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 756: this message will repeat every 5th retry.
W20260810 23:02:43.380479 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 23:02:43.710573 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 23:02:45.740887 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 761: this message will repeat every 5th retry.
W20260810 23:02:46.007972 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:02:46.008792 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 23:02:46.202714 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 23:02:48.411911 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 23:02:48.581427 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:48.582365 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 766: this message will repeat every 5th retry.
W20260810 23:02:48.620848 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 23:02:50.780568 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 23:02:51.092033 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:51.092823 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 23:02:51.131229 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 771: this message will repeat every 5th retry.
W20260810 23:02:53.168673 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 23:02:53.454963 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 23:02:53.710086 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:53.710896 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 776: this message will repeat every 5th retry.
W20260810 23:02:55.456225 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 23:02:56.372591 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:02:56.373485 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 23:02:56.394129 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 781: this message will repeat every 5th retry.
W20260810 23:02:57.759836 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 23:02:58.716938 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:02:58.717716 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 786: this message will repeat every 5th retry.
W20260810 23:02:58.778100 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 23:03:00.314713 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 23:03:01.180711 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 791: this message will repeat every 5th retry.
W20260810 23:03:01.258329 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 23:03:01.731472 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:03:02.902807 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 23:03:03.673519 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 796: this message will repeat every 5th retry.
W20260810 23:03:03.717237 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 23:03:04.279716 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:03:05.341360 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 23:03:06.209195 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 23:03:06.308382 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 801: this message will repeat every 5th retry.
W20260810 23:03:06.831578 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:03:07.540234 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 23:03:08.668447 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 806: this message will repeat every 5th retry.
W20260810 23:03:08.717777 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 23:03:09.705935 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:03:09.944309 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 23:03:10.904115 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 811: this message will repeat every 5th retry.
W20260810 23:03:11.097682 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 23:03:12.000393 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:03:12.421365 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 23:03:13.028419 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 816: this message will repeat every 5th retry.
W20260810 23:03:13.690511 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 23:03:14.894151 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 23:03:14.913314 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 11 similar messages]
W20260810 23:03:15.335995 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 821: this message will repeat every 5th retry.
W20260810 23:03:16.206476 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 23:03:17.102331 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:03:17.235339 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 23:03:18.040238 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 826: this message will repeat every 5th retry.
W20260810 23:03:18.467487 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 23:03:19.497505 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 23:03:19.958947 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:03:20.506567 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 831: this message will repeat every 5th retry.
W20260810 23:03:21.193919 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 23:03:21.345242 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (300 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:03:21.498853 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (301 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:03:22.180315 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:03:22.340554 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
W20260810 23:03:23.049026 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 836: this message will repeat every 5th retry.
W20260810 23:03:23.729943 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
W20260810 23:03:24.607379 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 23:03:24.980199 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:03:25.462839 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 841: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:03:25.674819 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:03:26.123375 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 23:03:27.276670 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:03:27.277740 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 23:03:28.277381 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 846: this message will repeat every 5th retry.
W20260810 23:03:28.665268 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 23:03:30.158275 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 23:03:30.482303 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:03:31.038266 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 23:03:31.105162 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 851: this message will repeat every 5th retry.
W20260810 23:03:32.564208 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:03:32.790256 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 23:03:33.411573 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 23:03:33.613252 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 856: this message will repeat every 5th retry.
W20260810 23:03:35.292135 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 23:03:35.536761 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:03:36.096779 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 861: this message will repeat every 5th retry.
W20260810 23:03:36.239039 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 23:03:37.699095 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:03:37.835662 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 23:03:38.458325 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 866: this message will repeat every 5th retry.
W20260810 23:03:38.627730 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 23:03:40.233600 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 23:03:40.897156 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:03:40.898101 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 871: this message will repeat every 5th retry.
W20260810 23:03:41.159752 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 23:03:42.634495 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 23:03:43.019181 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:03:43.440265 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 876: this message will repeat every 5th retry.
W20260810 23:03:43.491138 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 23:03:45.383000 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 23:03:45.820083 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 23:03:46.160490 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:03:46.161350 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 881: this message will repeat every 5th retry.
W20260810 23:03:47.864784 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 23:03:48.274124 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:03:48.378047 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 23:03:48.642902 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 886: this message will repeat every 5th retry.
W20260810 23:03:50.374028 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 23:03:50.738726 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 23:03:50.867810 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 891: this message will repeat every 5th retry.
W20260810 23:03:51.437834 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:03:53.073895 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 23:03:53.111850 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 896: this message will repeat every 5th retry.
W20260810 23:03:53.433236 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:03:53.434046 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 23:03:55.530722 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 901: this message will repeat every 5th retry.
W20260810 23:03:55.535811 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 23:03:55.644984 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 23:03:56.610122 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:03:58.010823 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 906: this message will repeat every 5th retry.
W20260810 23:03:58.103029 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 23:03:58.226444 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 23:03:58.529803 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:04:00.416718 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 911: this message will repeat every 5th retry.
W20260810 23:04:00.581010 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 23:04:00.640718 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 23:04:01.922679 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:02.959491 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 916: this message will repeat every 5th retry.
W20260810 23:04:03.030313 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 23:04:03.163189 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 23:04:03.546254 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:04:05.514600 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 23:04:05.517794 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 921: this message will repeat every 5th retry.
W20260810 23:04:05.839681 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 23:04:07.380708 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:07.994410 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 926: this message will repeat every 5th retry.
W20260810 23:04:08.047091 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 23:04:08.307371 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 23:04:08.567862 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:04:10.470666 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 931: this message will repeat every 5th retry.
W20260810 23:04:10.628504 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 23:04:10.764194 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 23:04:12.431773 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:04:12.858521 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 936: this message will repeat every 5th retry.
W20260810 23:04:13.341771 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 23:04:13.520035 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 23:04:13.768512 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:04:15.433703 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 941: this message will repeat every 5th retry.
W20260810 23:04:15.702843 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 23:04:15.993129 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 23:04:17.771925 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:17.772652 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 946: this message will repeat every 5th retry.
W20260810 23:04:18.350400 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 23:04:18.359725 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 23:04:18.873917 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:04:20.043172 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 951: this message will repeat every 5th retry.
W20260810 23:04:20.743276 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 23:04:20.952754 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 23:04:21.417815 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (360 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:04:21.570561 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (361 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:04:22.640681 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 956: this message will repeat every 5th retry.
W20260810 23:04:23.155936 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 23:04:23.190810 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:23.310339 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 23:04:24.220088 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:04:25.168381 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 961: this message will repeat every 5th retry.
W20260810 23:04:25.593453 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
W20260810 23:04:25.634760 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:04:26.887797 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:04:27.795421 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 966: this message will repeat every 5th retry.
W20260810 23:04:28.130056 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
W20260810 23:04:28.212235 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
W20260810 23:04:28.405503 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:04:29.652951 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:04:30.563498 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 971: this message will repeat every 5th retry.
W20260810 23:04:30.633698 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 23:04:30.781942 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 23:04:32.962365 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 23:04:33.200810 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 976: this message will repeat every 5th retry.
W20260810 23:04:33.246804 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 23:04:33.725453 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:04:34.863577 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:04:35.446141 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 23:04:35.792405 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 23:04:35.815680 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 981: this message will repeat every 5th retry.
W20260810 23:04:37.686455 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 23:04:38.108078 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 23:04:38.131785 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 986: this message will repeat every 5th retry.
W20260810 23:04:39.178829 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:39.967764 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:04:39.968529 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 23:04:40.538123 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 23:04:40.598775 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 991: this message will repeat every 5th retry.
W20260810 23:04:42.592203 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 23:04:42.762557 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 23:04:42.960649 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 996: this message will repeat every 5th retry.
W20260810 23:04:44.506713 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:45.024716 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:04:45.025522 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 23:04:45.145947 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 23:04:45.369452 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1001: this message will repeat every 5th retry.
W20260810 23:04:47.359978 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 23:04:47.681483 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 23:04:47.707743 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1006: this message will repeat every 5th retry.
W20260810 23:04:49.532086 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:49.714871 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 23:04:50.079643 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1011: this message will repeat every 5th retry.
W20260810 23:04:50.163359 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:04:50.164134 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 23:04:52.236393 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1016: this message will repeat every 5th retry.
W20260810 23:04:52.419112 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 23:04:52.490101 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 23:04:54.871294 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:04:54.872149 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1021: this message will repeat every 5th retry.
W20260810 23:04:55.228569 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:04:55.229322 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 23:04:55.503214 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 23:04:57.155609 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1026: this message will repeat every 5th retry.
W20260810 23:04:57.909417 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 23:04:58.047329 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 23:04:59.457322 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1031: this message will repeat every 5th retry.
W20260810 23:05:00.047111 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:05:00.280326 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:05:00.281134 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 23:05:00.501085 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 23:05:01.989953 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1036: this message will repeat every 5th retry.
W20260810 23:05:02.832087 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 23:05:02.968375 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 23:05:04.660714 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1041: this message will repeat every 5th retry.
W20260810 23:05:05.151885 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:05.264211 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 23:05:05.325466 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:05:05.326306 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 23:05:07.142907 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1046: this message will repeat every 5th retry.
W20260810 23:05:07.689850 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 23:05:07.785596 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 23:05:09.763216 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1051: this message will repeat every 5th retry.
W20260810 23:05:10.116169 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 23:05:10.285493 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:10.426680 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:05:10.427644 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 23:05:12.126597 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1056: this message will repeat every 5th retry.
W20260810 23:05:12.694481 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 23:05:12.962852 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 23:05:14.659070 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1061: this message will repeat every 5th retry.
W20260810 23:05:15.053798 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 23:05:15.287618 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 23:05:15.595805 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:05:15.625558 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:05:17.211830 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1066: this message will repeat every 5th retry.
W20260810 23:05:17.308696 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 23:05:17.912570 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 23:05:19.780067 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 23:05:19.954557 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1071: this message will repeat every 5th retry.
W20260810 23:05:20.115540 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 23:05:21.024796 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:21.025673 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 22 similar messages]
W20260810 23:05:21.490681 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (420 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:05:21.643445 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (421 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:05:22.083078 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 23:05:22.606837 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 23:05:22.650978 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1076: this message will repeat every 5th retry.
W20260810 23:05:24.631364 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
W20260810 23:05:24.906184 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
W20260810 23:05:25.123273 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1081: this message will repeat every 5th retry.
W20260810 23:05:26.235000 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:26.485325 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:05:26.991003 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
W20260810 23:05:27.535385 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
W20260810 23:05:27.892812 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1086: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:05:28.105918 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:05:29.580070 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 23:05:30.135524 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 23:05:30.374167 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1091: this message will repeat every 5th retry.
W20260810 23:05:31.473740 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:31.537159 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:05:31.955993 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 23:05:32.741124 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 23:05:32.848984 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1096: this message will repeat every 5th retry.
W20260810 23:05:34.320680 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 23:05:35.136335 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1101: this message will repeat every 5th retry.
W20260810 23:05:35.335803 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 23:05:36.619402 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:05:36.819177 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 23:05:36.984534 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 11 similar messages]
W20260810 23:05:37.447165 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1106: this message will repeat every 5th retry.
W20260810 23:05:37.538149 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 23:05:39.485365 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 23:05:40.055109 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 23:05:40.108662 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1111: this message will repeat every 5th retry.
W20260810 23:05:41.879808 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:05:41.880568 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 23:05:42.071797 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:42.393077 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 23:05:42.584349 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1116: this message will repeat every 5th retry.
W20260810 23:05:44.500777 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 23:05:44.727358 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 23:05:45.081159 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1121: this message will repeat every 5th retry.
W20260810 23:05:47.133800 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:05:47.252529 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:05:47.253281 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 23:05:47.366411 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 23:05:47.589536 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1126: this message will repeat every 5th retry.
W20260810 23:05:49.582528 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 23:05:49.670004 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 23:05:50.124951 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1131: this message will repeat every 5th retry.
W20260810 23:05:51.889281 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 23:05:51.988921 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 23:05:52.387104 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:05:52.407753 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:05:52.408620 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1136: this message will repeat every 5th retry.
W20260810 23:05:54.185279 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 23:05:54.516974 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 23:05:54.734416 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1141: this message will repeat every 5th retry.
W20260810 23:05:56.831306 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 23:05:57.007525 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1146: this message will repeat every 5th retry.
W20260810 23:05:57.032863 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 23:05:57.403590 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:05:57.434870 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:05:59.277137 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 23:05:59.420708 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1151: this message will repeat every 5th retry.
W20260810 23:05:59.664597 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 23:06:01.761857 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 23:06:01.931342 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1156: this message will repeat every 5th retry.
W20260810 23:06:02.251025 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 23:06:02.683060 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:06:02.819418 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:04.151537 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 23:06:04.342588 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 23:06:04.523749 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1161: this message will repeat every 5th retry.
W20260810 23:06:06.616667 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 23:06:06.648514 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 23:06:06.887969 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1166: this message will repeat every 5th retry.
W20260810 23:06:07.834379 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:06:08.073441 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 22 similar messages]
W20260810 23:06:08.996250 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 23:06:09.141252 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 23:06:09.430320 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1171: this message will repeat every 5th retry.
W20260810 23:06:11.314051 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 23:06:11.417642 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 23:06:11.755331 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1176: this message will repeat every 5th retry.
W20260810 23:06:13.142716 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:13.233109 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:06:13.807647 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 23:06:13.930260 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 23:06:14.256932 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1181: this message will repeat every 5th retry.
W20260810 23:06:16.302597 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 23:06:16.503239 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 23:06:17.071671 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1186: this message will repeat every 5th retry.
W20260810 23:06:18.266172 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 8 similar messages]
W20260810 23:06:18.455668 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:06:18.623693 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 23:06:18.952277 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 23:06:19.716995 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1191: this message will repeat every 5th retry.
W20260810 23:06:21.286144 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 23:06:21.562659 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (480 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:06:21.714432 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (481 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:06:21.734339 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 23:06:21.992514 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1196: this message will repeat every 5th retry.
W20260810 23:06:23.617197 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:23.811518 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:06:23.963604 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 23:06:24.335553 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 23:06:24.695310 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1201: this message will repeat every 5th retry.
W20260810 23:06:26.216751 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 23:06:26.893178 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 23:06:27.502595 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1206: this message will repeat every 5th retry.
W20260810 23:06:28.791266 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:06:28.883035 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:06:28.883935 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
W20260810 23:06:29.156677 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:06:29.297919 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:06:29.792935 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1211: this message will repeat every 5th retry.
W20260810 23:06:31.300282 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 23:06:31.759650 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 23:06:32.192607 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1216: this message will repeat every 5th retry.
W20260810 23:06:33.437232 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 23:06:33.905539 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:06:34.199172 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:34.429337 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 23:06:34.772051 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1221: this message will repeat every 5th retry.
W20260810 23:06:36.012351 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 23:06:36.850379 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 23:06:37.103880 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1226: this message will repeat every 5th retry.
W20260810 23:06:38.445232 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 23:06:38.937444 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:06:39.214638 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 23:06:39.451344 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:39.452137 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1231: this message will repeat every 5th retry.
W20260810 23:06:40.837103 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 23:06:41.581485 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 23:06:41.979880 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1236: this message will repeat every 5th retry.
W20260810 23:06:43.163750 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 23:06:44.258113 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:06:44.258996 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 23:06:44.614480 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:06:44.615533 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1241: this message will repeat every 5th retry.
W20260810 23:06:45.898821 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 23:06:47.037582 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 23:06:47.098608 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1246: this message will repeat every 5th retry.
W20260810 23:06:48.360088 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 23:06:49.282379 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:06:49.449043 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1251: this message will repeat every 5th retry.
W20260810 23:06:49.756642 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 23:06:49.860679 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:50.986255 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 23:06:51.929291 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1256: this message will repeat every 5th retry.
W20260810 23:06:52.343396 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 23:06:53.428331 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 23:06:54.391186 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:06:54.408102 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1261: this message will repeat every 5th retry.
W20260810 23:06:55.002710 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 23:06:55.254927 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:06:55.846329 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 23:06:56.901214 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1266: this message will repeat every 5th retry.
W20260810 23:06:57.438923 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 23:06:58.346717 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 23:06:59.273327 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1271: this message will repeat every 5th retry.
W20260810 23:06:59.521991 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:07:00.123664 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 23:07:00.417045 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:00.749536 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 23:07:01.844946 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1276: this message will repeat every 5th retry.
W20260810 23:07:02.787504 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 23:07:03.216282 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 23:07:04.541245 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1281: this message will repeat every 5th retry.
W20260810 23:07:04.574941 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:07:05.299446 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 23:07:05.678887 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:05.758301 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 23:07:07.023460 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1286: this message will repeat every 5th retry.
W20260810 23:07:07.824870 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 23:07:08.272151 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 23:07:09.516902 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1291: this message will repeat every 5th retry.
W20260810 23:07:09.580783 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:07:10.374085 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 23:07:10.486245 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 23:07:11.070700 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:11.993353 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1296: this message will repeat every 5th retry.
W20260810 23:07:12.989786 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 23:07:13.003859 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 23:07:14.472951 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1301: this message will repeat every 5th retry.
W20260810 23:07:14.620913 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:07:15.624804 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 23:07:15.652905 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 23:07:16.419384 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:16.823683 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1306: this message will repeat every 5th retry.
W20260810 23:07:18.152495 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 23:07:18.234664 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 23:07:19.158574 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1311: this message will repeat every 5th retry.
W20260810 23:07:19.888983 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:07:20.981509 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 23:07:21.191146 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 23:07:21.633358 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (540 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:07:21.788100 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (541 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:07:21.815847 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:21.816828 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1316: this message will repeat every 5th retry.
W20260810 23:07:23.528980 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 23:07:23.630182 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 23:07:24.079954 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1321: this message will repeat every 5th retry.
W20260810 23:07:25.099048 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:07:25.989003 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 23:07:26.227118 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 23:07:26.694039 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1326: this message will repeat every 5th retry.
W20260810 23:07:27.277940 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:28.578925 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 23:07:28.644040 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 23:07:29.483831 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1331: this message will repeat every 5th retry.
W20260810 23:07:30.117959 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:07:30.492390 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:07:31.071951 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 23:07:31.172583 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
W20260810 23:07:32.157119 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1336: this message will repeat every 5th retry.
W20260810 23:07:32.743372 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:33.465430 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 23:07:33.608564 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
W20260810 23:07:34.722404 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1341: this message will repeat every 5th retry.
W20260810 23:07:35.210362 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:07:35.884382 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 23:07:36.045686 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 23:07:37.598889 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1346: this message will repeat every 5th retry.
W20260810 23:07:38.070276 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:38.525982 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 23:07:38.545010 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 23:07:40.096063 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1351: this message will repeat every 5th retry.
W20260810 23:07:40.406253 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:07:40.831888 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 23:07:40.861907 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 23:07:42.607486 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1356: this message will repeat every 5th retry.
W20260810 23:07:43.110354 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:43.267467 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 23:07:43.436040 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 23:07:45.218777 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1361: this message will repeat every 5th retry.
W20260810 23:07:45.564684 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:07:45.565603 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 23:07:46.172946 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 23:07:47.996313 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1366: this message will repeat every 5th retry.
W20260810 23:07:48.203254 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 23:07:48.438915 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:07:48.526959 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 23:07:50.494477 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 23:07:50.506096 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1371: this message will repeat every 5th retry.
W20260810 23:07:50.961292 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:07:50.962174 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 23:07:52.683662 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 23:07:52.908876 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1376: this message will repeat every 5th retry.
W20260810 23:07:53.577566 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 23:07:53.788825 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:55.160812 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1381: this message will repeat every 5th retry.
W20260810 23:07:55.370460 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 23:07:55.969089 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:07:55.970036 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 23:07:57.635336 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1386: this message will repeat every 5th retry.
W20260810 23:07:57.849488 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 23:07:58.807258 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 23:07:58.883800 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:07:59.805986 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1391: this message will repeat every 5th retry.
W20260810 23:08:00.252014 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 23:08:01.219602 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:08:01.233330 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 23:08:02.409094 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1396: this message will repeat every 5th retry.
W20260810 23:08:02.809899 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 23:08:03.673326 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 23:08:04.334261 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:04.766417 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1401: this message will repeat every 5th retry.
W20260810 23:08:05.465335 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 23:08:06.102022 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 23:08:06.421700 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:06.939738 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1406: this message will repeat every 5th retry.
W20260810 23:08:07.824795 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 23:08:08.582083 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 23:08:09.546926 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:09.547835 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1411: this message will repeat every 5th retry.
W20260810 23:08:10.240686 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 23:08:11.028198 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 23:08:11.496913 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:12.011551 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1416: this message will repeat every 5th retry.
W20260810 23:08:12.891726 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 23:08:13.429768 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 23:08:14.819371 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:08:14.820278 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1421: this message will repeat every 5th retry.
W20260810 23:08:15.588328 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 23:08:16.119259 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 23:08:16.558877 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:08:17.465858 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1426: this message will repeat every 5th retry.
W20260810 23:08:18.102492 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 23:08:18.695080 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 23:08:19.988050 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:08:19.988963 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1431: this message will repeat every 5th retry.
W20260810 23:08:20.462024 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 23:08:21.283389 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 23:08:21.706033 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (600 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:08:21.799796 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:21.860291 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (601 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:08:22.252923 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1436: this message will repeat every 5th retry.
W20260810 23:08:22.920668 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 23:08:23.898154 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 23:08:24.614413 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1441: this message will repeat every 5th retry.
W20260810 23:08:25.131098 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:25.136855 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 23:08:26.299248 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 23:08:26.847213 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:26.930390 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1446: this message will repeat every 5th retry.
W20260810 23:08:27.714038 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 23:08:28.804286 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 23:08:29.445910 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1451: this message will repeat every 5th retry.
W20260810 23:08:30.209832 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 23:08:30.415812 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:31.436290 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:08:31.680843 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:08:31.967492 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:08:32.071836 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1456: this message will repeat every 5th retry.
W20260810 23:08:32.790891 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
W20260810 23:08:33.882623 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 23:08:34.447175 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1461: this message will repeat every 5th retry.
W20260810 23:08:35.163018 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 23:08:35.607039 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:08:36.395673 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 23:08:37.077065 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1466: this message will repeat every 5th retry.
W20260810 23:08:37.318511 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:37.695497 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 23:08:38.853901 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 23:08:39.524462 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1471: this message will repeat every 5th retry.
W20260810 23:08:40.276208 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 23:08:40.860674 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:41.311947 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 23:08:41.807132 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1476: this message will repeat every 5th retry.
W20260810 23:08:42.792618 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:08:42.793394 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 23:08:43.979735 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 23:08:44.048437 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1481: this message will repeat every 5th retry.
W20260810 23:08:45.145709 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 23:08:45.872131 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:46.391909 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1486: this message will repeat every 5th retry.
W20260810 23:08:46.733880 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 23:08:47.577841 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 23:08:47.820576 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:08:49.022521 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1491: this message will repeat every 5th retry.
W20260810 23:08:49.429401 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 23:08:50.246857 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 23:08:51.245488 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:51.246492 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1496: this message will repeat every 5th retry.
W20260810 23:08:51.784287 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 23:08:52.420883 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 23:08:52.913278 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:08:53.593214 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1501: this message will repeat every 5th retry.
W20260810 23:08:54.100600 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 23:08:54.929193 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 23:08:56.124688 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1506: this message will repeat every 5th retry.
W20260810 23:08:56.428917 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 23:08:56.548326 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:08:57.678800 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 23:08:58.051208 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:08:58.638100 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1511: this message will repeat every 5th retry.
W20260810 23:08:58.950248 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 23:08:59.946496 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 23:09:01.246349 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1516: this message will repeat every 5th retry.
W20260810 23:09:01.371168 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 23:09:01.798156 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:09:02.538275 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 23:09:03.097227 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:09:03.761251 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1521: this message will repeat every 5th retry.
W20260810 23:09:03.976080 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 23:09:05.103093 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 23:09:06.393810 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1526: this message will repeat every 5th retry.
W20260810 23:09:06.610016 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 23:09:06.930930 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:09:07.504642 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 23:09:08.486662 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:09:08.817139 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1531: this message will repeat every 5th retry.
W20260810 23:09:09.084733 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 23:09:09.864053 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 23:09:11.123034 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1536: this message will repeat every 5th retry.
W20260810 23:09:11.775859 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 23:09:11.984743 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 23:09:12.132426 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:13.642822 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1541: this message will repeat every 5th retry.
W20260810 23:09:13.933848 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:09:14.467599 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 23:09:14.530535 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 23:09:16.260391 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1546: this message will repeat every 5th retry.
W20260810 23:09:17.081241 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 23:09:17.162349 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 23:09:17.585021 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:18.603554 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1551: this message will repeat every 5th retry.
W20260810 23:09:19.142141 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:09:19.759092 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 23:09:19.851576 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 23:09:21.151814 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1556: this message will repeat every 5th retry.
W20260810 23:09:21.778191 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (660 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:09:21.933357 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (661 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:09:22.119614 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 23:09:22.536361 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 23:09:23.010064 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:23.603340 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1561: this message will repeat every 5th retry.
W20260810 23:09:24.403774 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:09:24.446638 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 23:09:24.830665 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 23:09:26.069550 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1566: this message will repeat every 5th retry.
W20260810 23:09:26.928437 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 23:09:27.360797 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 23:09:28.083227 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:09:28.666810 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1571: this message will repeat every 5th retry.
W20260810 23:09:29.144178 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
W20260810 23:09:29.593590 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:09:30.054395 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
W20260810 23:09:31.341809 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1576: this message will repeat every 5th retry.
W20260810 23:09:31.539117 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
W20260810 23:09:32.834111 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:09:32.921839 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:09:33.421247 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:09:33.808846 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1581: this message will repeat every 5th retry.
W20260810 23:09:34.036444 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 23:09:34.897174 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 21 similar messages]
W20260810 23:09:35.170730 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 23:09:36.256382 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1586: this message will repeat every 5th retry.
W20260810 23:09:36.597018 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 23:09:37.648041 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 23:09:38.721029 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:38.721876 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1591: this message will repeat every 5th retry.
W20260810 23:09:39.201007 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 23:09:39.975178 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 18 similar messages]
W20260810 23:09:40.446041 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 23:09:41.271111 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1596: this message will repeat every 5th retry.
W20260810 23:09:41.724120 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 23:09:43.249989 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 23:09:43.973219 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:09:43.974161 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1601: this message will repeat every 5th retry.
W20260810 23:09:44.084817 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 23:09:45.202145 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:09:45.751596 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 23:09:46.551232 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1606: this message will repeat every 5th retry.
W20260810 23:09:46.861503 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 23:09:48.379449 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 23:09:48.899741 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1611: this message will repeat every 5th retry.
W20260810 23:09:49.295567 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:49.463420 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 23:09:50.444383 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:09:51.095373 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 23:09:51.373018 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1616: this message will repeat every 5th retry.
W20260810 23:09:52.110646 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 23:09:53.617961 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 23:09:53.763456 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1621: this message will repeat every 5th retry.
W20260810 23:09:54.588672 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 23:09:54.734944 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:09:55.576401 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:09:56.174258 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 23:09:56.481017 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1626: this message will repeat every 5th retry.
W20260810 23:09:57.261816 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 23:09:58.796792 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 23:09:58.842114 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1631: this message will repeat every 5th retry.
W20260810 23:09:59.844336 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 23:09:59.930845 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:00.719640 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:01.423350 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 23:10:01.524240 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1636: this message will repeat every 5th retry.
W20260810 23:10:02.232443 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 23:10:03.659803 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 23:10:03.811235 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1641: this message will repeat every 5th retry.
W20260810 23:10:04.828681 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 23:10:05.372035 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:10:05.826179 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:06.164829 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 23:10:06.421154 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1646: this message will repeat every 5th retry.
W20260810 23:10:07.387029 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 23:10:08.554083 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 23:10:09.030179 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1651: this message will repeat every 5th retry.
W20260810 23:10:10.005844 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 23:10:10.604516 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:10:11.014395 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:11.173575 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 23:10:11.572362 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1656: this message will repeat every 5th retry.
W20260810 23:10:12.580034 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 23:10:13.984555 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 23:10:14.079890 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1661: this message will repeat every 5th retry.
W20260810 23:10:15.040824 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 23:10:15.930017 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:16.049332 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:16.495227 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 23:10:16.628104 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1666: this message will repeat every 5th retry.
W20260810 23:10:17.583775 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 23:10:18.891839 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 23:10:18.994544 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1671: this message will repeat every 5th retry.
W20260810 23:10:20.238528 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 23:10:21.046159 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:21.183029 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:10:21.538873 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 23:10:21.567551 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1676: this message will repeat every 5th retry.
W20260810 23:10:21.850939 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (720 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:10:22.004077 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (721 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:10:22.651046 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 23:10:24.006933 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 23:10:24.079591 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1681: this message will repeat every 5th retry.
W20260810 23:10:24.901371 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 23:10:26.069126 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:26.235596 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:26.522238 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 23:10:26.706485 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1686: this message will repeat every 5th retry.
W20260810 23:10:27.284966 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 23:10:28.903658 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 23:10:29.131772 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1691: this message will repeat every 5th retry.
W20260810 23:10:29.643508 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 23:10:31.196681 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:31.286204 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:10:31.671690 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 23:10:31.878961 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1696: this message will repeat every 5th retry.
W20260810 23:10:32.078912 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
W20260810 23:10:33.985051 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
Master Summary
UUID | Address | Status
---------------------------------+---------------------+-------------
<unknown> (127.6.106.190:36779) | 127.6.106.190:36779 | UNAVAILABLE
Error from 127.6.106.190:36779: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111) (UNAVAILABLE)
All reported replicas are:
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+----------+--------------+--------------+------------
Tablet Server Summary
Version Summary
Version | Servers
---------+---------
The cluster doesn't have any matching tablets
The cluster doesn't have any matching system tables
The cluster doesn't have any matching tables
==================
Warnings:
==================
master unusual flags check error: 1 of 1 masters were not available to retrieve unusual flags
master diverged flags check error: 1 of 1 masters were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from masters: failed to gather info from all masters: 1 of 1 had errors
Not found: master consensus error: no master consensus state available
Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
FAILED
I20260810 23:10:34.148762 6570 cluster_verifier.cc:83] Check not successful yet, sleeping and retrying: Timed out: leader master liveness check error: Could not connect to the cluster: ConnectToClusterRpc(addrs: 127.6.106.190:36779, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:10:34.371114 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 23:10:34.576606 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1701: this message will repeat every 5th retry.
W20260810 23:10:36.300314 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:36.502187 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:36.811954 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 23:10:36.943009 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 23:10:37.194895 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1706: this message will repeat every 5th retry.
W20260810 23:10:39.128553 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 23:10:39.455600 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 23:10:39.520787 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1711: this message will repeat every 5th retry.
W20260810 23:10:41.495505 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:41.724661 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:10:41.801849 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 23:10:42.048609 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 23:10:42.136526 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1716: this message will repeat every 5th retry.
W20260810 23:10:44.187063 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 23:10:44.665791 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 23:10:44.780375 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1721: this message will repeat every 5th retry.
W20260810 23:10:46.608853 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
W20260810 23:10:46.631901 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:10:46.831269 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:10:47.065009 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 23:10:47.272776 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1726: this message will repeat every 5th retry.
W20260810 23:10:49.121161 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
W20260810 23:10:49.567677 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
W20260810 23:10:49.907023 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1731: this message will repeat every 5th retry.
************************ BEGIN STACKS **************************
[New LWP 6573]
[New LWP 6574]
[New LWP 6575]
[New LWP 7100]
[New LWP 7103]
[New LWP 7104]
[New LWP 7105]
[New LWP 12368]
[New LWP 12369]
[New LWP 12370]
[New LWP 12371]
[New LWP 12374]
[New LWP 12375]
[New LWP 12376]
[New LWP 12377]
[New LWP 12378]
W20260810 23:10:51.631981 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 23:10:51.982234 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:10:51.983079 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
W20260810 23:10:52.064198 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:10:52.670872 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1736: this message will repeat every 5th retry.
W20260810 23:10:54.049203 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 23:10:54.355360 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 23:10:55.147554 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1741: this message will repeat every 5th retry.
W20260810 23:10:56.775817 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 23:10:56.860244 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 23:10:57.281087 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:10:57.313486 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:10:57.314630 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1746: this message will repeat every 5th retry.
W20260810 23:10:59.107497 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 23:10:59.356994 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 23:10:59.646919 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1751: this message will repeat every 5th retry.
W20260810 23:11:01.487295 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 23:11:01.774300 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 23:11:01.981439 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1756: this message will repeat every 5th retry.
W20260810 23:11:02.366263 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:11:02.429484 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:11:04.104236 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 23:11:04.351095 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 23:11:04.524137 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1761: this message will repeat every 5th retry.
W20260810 23:11:06.503475 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 23:11:06.800292 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 23:11:07.040230 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1766: this message will repeat every 5th retry.
W20260810 23:11:07.424213 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 19 similar messages]
W20260810 23:11:07.773284 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 10 similar messages]
W20260810 23:11:08.761474 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 23:11:09.475054 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
W20260810 23:11:09.675601 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1771: this message will repeat every 5th retry.
W20260810 23:11:11.386215 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
W20260810 23:11:11.716116 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
W20260810 23:11:12.290495 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1776: this message will repeat every 5th retry.
W20260810 23:11:12.692875 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:11:12.801455 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:11:13.762112 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
W20260810 23:11:14.316347 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
W20260810 23:11:14.821075 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1781: this message will repeat every 5th retry.
W20260810 23:11:16.391498 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
W20260810 23:11:16.597954 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
W20260810 23:11:17.642307 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1786: this message will repeat every 5th retry.
W20260810 23:11:17.904192 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:11:18.142223 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
W20260810 23:11:18.738160 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
0x00007fe74a28cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7fe7448da240 (LWP 6570) "all_types-itest" 0x00007fe74a28cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7fe74407c700 (LWP 6573) "all_types-itest" 0x00007fe74a29132a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7fe742ab3700 (LWP 6574) "kernel-watcher-" 0x00007fe74a28cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7fe7422b2700 (LWP 6575) "rpc reactor-657" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7fe741ab1700 (LWP 7100) "rpc reactor-710" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7fe7410a9700 (LWP 7103) "rpc reactor-710" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7fe7408a8700 (LWP 7104) "rpc reactor-710" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7fe7400a7700 (LWP 7105) "rpc reactor-710" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7fe73c09f700 (LWP 12368) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7fe73a09b700 (LWP 12369) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7fe739099700 (LWP 12370) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7fe73d0a1700 (LWP 12371) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7fe73e8a4700 (LWP 12374) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7fe73e0a3700 (LWP 12375) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7fe73d8a2700 (LWP 12376) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7fe73c8a0700 (LWP 12377) "rpc reactor-123" 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7fe73b89e700 (LWP 12378) "client-negotiat" 0x00007fe74a28cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 17 (Thread 0x7fe73b89e700 (LWP 12378)):
#0 0x00007fe74a28cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
W20260810 23:11:19.262957 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
#1 0x00007fe749031de9 in kudu::ConditionVariable::WaitFor (this=0x7fe73b89d140, delta=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:121
#2 0x00007fe749166ecf in kudu::ThreadPool::DispatchThread (this=0x299e5180) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:721
#3 0x00007fe749167c83 in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x2a01e028) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/threadpool.cc:849
W20260810 23:11:19.844214 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1791: this message will repeat every 5th retry.
#4 0x00007fe7491693f2 in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#5 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x2a01e028) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#6 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x2a01e000) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#7 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7fe73c8a0700 (LWP 12377)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925df80, timeout=0.099256332498043776) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925df80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x291938a8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x291938a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c7618) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c7618) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c75f0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7fe73d8a2700 (LWP 12376)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925f180, timeout=0.099705252563580871) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925f180, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29193228, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29193220) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c7478) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c7478) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c7450) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7fe73e0a3700 (LWP 12375)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925d200, timeout=0.098990866681560874) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925d200, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29193708, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29193700) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c7a28) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c7a28) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c7a00) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7fe73e8a4700 (LWP 12374)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925fa80, timeout=0.099213467445224524) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925fa80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x292e01e8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x292e01e0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c72d8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c72d8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c72b0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7fe73d0a1700 (LWP 12371)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925ed00, timeout=0.09988527256064117) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925ed00, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x292e1088, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x292e1080) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c7138) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c7138) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c7110) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7fe739099700 (LWP 12370)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925db00, timeout=0.099874455481767654) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925db00, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x292e0ee8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x292e0ee0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x290c7c98) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c7c98) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c7c70) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7fe73a09b700 (LWP 12369)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x2925d680, timeout=0.098926046630367637) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x2925d680, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x292e0d48, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x292e0d40) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c5d68) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c5d68) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c5d40) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7fe73c09f700 (LWP 12368)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29206d00, timeout=0.099980610422790051) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29206d00, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x292e0ba8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x292e0ba0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c4df8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c4df8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c4dd0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7fe7400a7700 (LWP 7105)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29205680, timeout=0.099858594825491309) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29205680, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29193088, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29193080) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c56e8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c56e8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c56c0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7fe7408a8700 (LWP 7104)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29205200, timeout=0.099196995375677943) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29205200, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29192ee8, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29192ee0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c5618) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c5618) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c55f0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7fe7410a9700 (LWP 7103)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29204d80, timeout=0.09985087881796062) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29204d80, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29192a08, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29192a00) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c5548) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c5548) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c5520) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7fe741ab1700 (LWP 7100)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29204900, timeout=0.099818340968340635) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29204900, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29192d48, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29192d40) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c5478) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c5478) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c5450) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7fe7422b2700 (LWP 6575)):
#0 0x00007fe7456ab947 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe7480e6bda in epoll_poll (loop=0x29204480, timeout=0.099068514071404934) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe7480ea228 in ev_run (loop=0x29204480, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe74d3a4207 in ev::loop_ref::run (this=0x29192388, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe74d39f34a in kudu::rpc::ReactorThread::RunThread (this=0x29192380) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:510
#5 0x00007fe74d39bfb5 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x291c4368) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/reactor.cc:197
#6 0x00007fe74d3a344d in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x291c4368) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x291c4340) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7fe742ab3700 (LWP 6574)):
#0 0x00007fe74a28cfb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe749031c38 in kudu::ConditionVariable::WaitUntil (this=0x29077878, until=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:87
#2 0x00007fe751be94e1 in kudu::CountDownLatch::WaitUntil (this=0x29077840, when=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:89
#3 0x00007fe751be9569 in kudu::CountDownLatch::WaitFor (this=0x29077840, delta=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99
#4 0x00007fe7490bdf57 in kudu::KernelStackWatchdog::RunThread (this=0x290777a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007fe7490bd291 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x290c6ec8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007fe7490be7f3 in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:297
#7 0x00007fe751fa0170 in std::function<void ()>::operator()() const (this=0x290c6ec8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/std_function.h:687
#8 0x00007fe749158512 in kudu::Thread::SuperviseThread (arg=0x290c6ea0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694
#9 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7fe74407c700 (LWP 6573)):
#0 0x00007fe74a29132a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe74914fe73 in kudu::Subprocess::DoWait (this=0x7fe74407b2a0, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/subprocess.cc:817
#2 0x00007fe74914d433 in kudu::Subprocess::Wait (this=0x7fe74407b2a0, wait_status=0x0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/subprocess.cc:544
#3 0x00007fe7490ac869 in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:258
#4 0x00007fe7490ac418 in kudu::PstackWatcher::RunGdbStackDump (pid=6570, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:218
#5 0x00007fe7490abd1a in kudu::PstackWatcher::DumpPidStacks (pid=6570, flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:169
#6 0x00007fe7490abcb5 in kudu::PstackWatcher::DumpStacks (flags=0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/pstack_watcher.cc:162
#7 0x0000000000513498 in kudu::<lambda()>::operator()(void) const (__closure=0x2909fbc8) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:60
#8 0x0000000000513bae in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60
#9 0x0000000000513960 in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:95
#10 0x0000000000514180 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x2909fbc8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244
#11 0x000000000051412a in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x2909fbc8) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253
#12 0x00000000005140f4 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x2909fbc0) at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196
#13 0x00007fe74df9fa4f in execute_native_thread_routine () from /tmp/dist-test-taskC98JjP/build/debug/bin/../../../thirdparty/installed/uninstrumented/lib/librocksdb.so.7
#14 0x00007fe74a2866db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007fe7456ab61f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7fe7448da240 (LWP 6570)):
#0 0x00007fe74a28cad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
W20260810 23:11:21.385442 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1801: this message will repeat every 5th retry.
#1 0x00007fe749031ade in kudu::ConditionVariable::Wait (this=0x295db590) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/condition_variable.cc:55
W20260810 23:11:21.710475 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
#2 0x00007fe7523b7114 in kudu::CountDownLatch::Wait (this=0x295db558) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:79
W20260810 23:11:21.925894 6816 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (780 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:11:22.081326 6958 heartbeater.cc:644] Failed to heartbeat to 127.6.106.190:36779 (781 consecutive failures): Network error: Failed to ping master at 127.6.106.190:36779: Client connection negotiation failed: client connection to 127.6.106.190:36779: connect: Connection refused (error 111)
W20260810 23:11:22.252184 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1796: this message will repeat every 5th retry.
#3 0x00007fe751ed552a in kudu::Synchronizer::Wait (this=0x7ffcab255510) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/async_util.h:58
#4 0x00007fe74ffa2d7d in kudu::client::KuduClient::Data::ConnectToCluster (this=0x29375860, client=0x2a4c6de0, deadline=..., creds_policy=kudu::rpc::CredentialsPolicy::ANY_CREDENTIALS) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/client/client-internal.cc:815
W20260810 23:11:23.113437 6837 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 20 similar messages]
W20260810 23:11:23.181550 6700 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111) [suppressed 9 similar messages]
#5 0x00007fe74ff680e4 in kudu::client::KuduClientBuilder::Build (this=0x7ffcab2558c8, client=0x2a0986b0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/client/client.cc:469
#6 0x00007fe750287bd1 in kudu::tools::CreateKuduClient (master_addresses=std::vector of length 1, capacity 1 = {...}, client=0x2a0986b0, can_see_all_replicas=true) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/tool_action_common.cc:738
W20260810 23:11:24.192080 6837 consensus_peers.cc:487] T eb23104048554e98ab96128e229998cc P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1806: this message will repeat every 5th retry.
W20260810 23:11:24.369513 6700 consensus_peers.cc:487] T 4fc5146ddfff440594fd98d716c8bf1c P d8411147a1174ff8aeadf0e21a8edfc5 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1801: this message will repeat every 5th retry.
#7 0x00007fe751bffc9d in kudu::tools::RemoteKsckCluster::Connect (this=0x2a0985a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck_remote.cc:530
#8 0x00007fe751bbd33f in kudu::tools::Ksck::CheckClusterRunning (this=0x299ad810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:460
#9 0x00007fe751bbe80a in kudu::tools::Ksck::Run (this=0x299ad810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:595
#10 0x00007fe751bc0a86 in kudu::tools::Ksck::RunAndPrintResults (this=0x299ad810) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/tools/ksck.cc:801
#11 0x00007fe752389640 in kudu::ClusterVerifier::RunKsck (this=0x7ffcab2560a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/cluster_verifier.cc:113
#12 0x00007fe7523890a5 in kudu::ClusterVerifier::CheckCluster (this=0x7ffcab2560a0) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/cluster_verifier.cc:78
W20260810 23:11:25.024823 6837 consensus_peers.cc:487] T d4f81674db9541b88bfb97fefcc225ff P 131b3fbf5d0048889a033c7aad167c31 -> Peer 48d765d5719f4b159d095ecafc56cea3 (127.6.106.131:36609): Couldn't send request to peer 48d765d5719f4b159d095ecafc56cea3. Status: Network error: Client connection negotiation failed: client connection to 127.6.106.131:36609: connect: Connection refused (error 111). This is attempt 1801: this message will repeat every 5th retry.
#13 0x00000000004d06d5 in kudu::client::AllTypesItest<kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> > >::RunTest(std::function<kudu::Status (kudu::client::KuduScanner*)> const&, std::function<void (kudu::client::KuduScanBatch const&, int, int*)> const&) (this=0x29194900, scanner_setup=..., verifier=...) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/all_types-itest.cc:622
#14 0x00000000004a451c in kudu::client::AllTypesItest_TestTimestampPadding_Test<kudu::client::IntKeysTestSetup<kudu::client::KeyTypeWrapper<(kudu::DataType)3> > >::TestBody (this=0x29194900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/integration-tests/all_types-itest.cc:804
#15 0x00007fe7493e9bfe in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7fe7493efa28 "the test body", method=<optimized out>, object=0x29194900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:831
#16 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x29194900, method=<optimized out>, location=location@entry=0x7fe7493efa28 "the test body") at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#17 0x00007fe7493de5db in testing::Test::Run (this=0x29194900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#18 testing::Test::Run (this=0x29194900) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2664
#19 0x00007fe7493de795 in testing::TestInfo::Run (this=0x290c6340) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#20 0x00007fe7493decdf in testing::TestSuite::Run (this=0x29176000) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#21 testing::TestSuite::Run (this=0x29176000) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2985
#22 0x00007fe7493df29f in testing::internal::UnitTestImpl::RunAllTests() () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:600
#23 0x00007fe7493ea13e in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7fe7493f15a8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x290b2280) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest-internal-inl.h:831
#24 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x290b2280, method=<optimized out>, location=location@entry=0x7fe7493f15a8 "auxiliary test code (environments or event listeners)") at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#25 0x00007fe7493de83d in testing::UnitTest::Run() () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/src/googletest-release-1.12.1/googletest/include/gtest/gtest.h:1246
#26 0x00000000005144b0 in RUN_ALL_TESTS () at /data0/somelongdirectorytoavoidrpathissues/src/kudu/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#27 0x00000000005138da in main (argc=1, argv=0x7ffcab256608) at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:109
************************* END STACKS ***************************
F20260810 23:11:25.480098 6573 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1786403485 (unix time) try "date -d @1786403485" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e8000019aa) received by PID 6570 (TID 0x7fe74407c700) from PID 6570; stack trace: ***
@ 0x7fe747b13a46 google::(anonymous namespace)::FailureSignalHandler() at ??:0
@ 0x7fe74a291980 (unknown) at ??:0
@ 0x7fe7455c8e87 gsignal at ??:0
@ 0x7fe7455ca7f1 abort at ??:0
@ 0x7fe747b059ad google::LogMessage::Fail() at ??:0
@ 0x7fe747b0888a google::LogMessage::SendToLog() at ??:0
@ 0x7fe747b054de google::LogMessage::Flush() at ??:0
@ 0x7fe747b06a19 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x5135cc _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/test_main.cc:63 (discriminator 9)
@ 0x513bae _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:60
@ 0x513960 _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at /opt/rh/devtoolset-8/root/usr/include/c++/8/bits/invoke.h:96
@ 0x514180 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:244
@ 0x51412a _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:253
@ 0x5140f4 _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at /opt/rh/devtoolset-8/root/usr/include/c++/8/thread:196
@ 0x7fe74df9fa4f execute_native_thread_routine at ??:0
@ 0x7fe74a2866db start_thread at ??:0
@ 0x7fe7456ab61f clone at ??:0