[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from RaftConsensusStressITest
[ RUN ] RaftConsensusStressITest.RemoveReplaceInCycle
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 20:45:27.866643 10079 test_util.cc:268] Using random seed: -1360682752
I20240502 20:45:27.873239 10079 ts_itest-base.cc:112] Starting cluster with:
I20240502 20:45:27.873301 10079 ts_itest-base.cc:113] --------------
I20240502 20:45:27.873319 10079 ts_itest-base.cc:114] 6 tablet servers
I20240502 20:45:27.873340 10079 ts_itest-base.cc:115] 3 replicas per TS
I20240502 20:45:27.873360 10079 ts_itest-base.cc:116] --------------
2024-05-02T20:45:27Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T20:45:27Z Disabled control of system clock
I20240502 20:45:27.883077 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.9.215.254:38205
--webserver_interface=127.9.215.254
--webserver_port=0
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.9.215.254:38205
--minloglevel=2
--raft_prepare_replacement_before_eviction=true
--max_create_tablets_per_ts=40 with env {}
I20240502 20:45:28.068825 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10091
I20240502 20:45:28.068957 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 20:45:28.138566 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.193:0
--local_ip_for_outbound_sockets=127.9.215.193
--webserver_interface=127.9.215.193
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:28.314662 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10186
I20240502 20:45:28.314754 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 20:45:28.317278 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.194:0
--local_ip_for_outbound_sockets=127.9.215.194
--webserver_interface=127.9.215.194
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:28.503868 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10321
I20240502 20:45:28.504029 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 20:45:28.506650 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.195:0
--local_ip_for_outbound_sockets=127.9.215.195
--webserver_interface=127.9.215.195
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:28.703102 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10454
I20240502 20:45:28.703240 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 20:45:28.705430 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.196:0
--local_ip_for_outbound_sockets=127.9.215.196
--webserver_interface=127.9.215.196
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:28.892171 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10588
I20240502 20:45:28.892305 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 20:45:28.894459 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.197:0
--local_ip_for_outbound_sockets=127.9.215.197
--webserver_interface=127.9.215.197
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:29.080528 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10723
I20240502 20:45:29.080641 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 20:45:29.082876 10079 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
/tmp/dist-test-taskZRhIiY/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.9.215.198:0
--local_ip_for_outbound_sockets=127.9.215.198
--webserver_interface=127.9.215.198
--webserver_port=0
--tserver_master_addrs=127.9.215.254:38205
--builtin_ntp_servers=127.9.215.212:38961
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 20:45:29.268815 10079 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskZRhIiY/build/debug/bin/kudu as pid 10857
I20240502 20:45:29.268916 10079 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 20:45:29.283865 10079 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 20:45:29.379325 10079 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 20:45:29.381004 10079 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 841ade2081574e7f9e2b047a7f94550e to finish bootstrapping
I20240502 20:45:29.387284 10079 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 1bde2b26ef9b45e38c446a1a41d7c0de to finish bootstrapping
I20240502 20:45:29.392485 10079 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 168ea3788aeb40419f31ec54a55d9ba6 to finish bootstrapping
I20240502 20:45:29.397768 10079 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 564db356872644cca9367376ee209e1f to finish bootstrapping
I20240502 20:45:29.402774 10079 ts_itest-base.cc:233] Waiting for 1 tablets on tserver cc70668e94e44fe9b3f7ea316962c766 to finish bootstrapping
I20240502 20:45:29.407460 10079 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 749a9092579845c28b8b65b24a5520c8 to finish bootstrapping
I20240502 20:45:29.412086 10079 test_util.cc:268] Using random seed: -1359137300
F20240502 20:45:40.090770 11675 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.queue_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.queue_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_cpu_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_cpu_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_wall_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_wall_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.queue_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.queue_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_cpu_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_cpu_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_wall_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_wall_time_us) (tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.queue_time_us, tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.queue_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.queue_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.queue_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_cpu_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_cpu_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_wall_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_wall_time_us) (tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.queue_time_us, tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.queue_time_us) (tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.run_cpu_time_us, tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.run_cpu_time_us) (tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.run_wall_time_us, tablet-download-pool-439d311e99ff408aa2b8fee445449d6c.run_wall_time_us) (tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.queue_time_us, tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.queue_time_us) (tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.run_cpu_time_us, tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.run_cpu_time_us) (tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.run_wall_time_us, tablet-download-pool-5d2d226d2dd74c12aa21d7f2058ecec5.run_wall_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.queue_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.queue_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_cpu_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_cpu_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_wall_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_wall_time_us) (tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us, tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us) (tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.run_cpu_time_us, tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.run_cpu_time_us) (tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.run_wall_time_us, tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f061f7aadcd google::LogMessage::Fail() at ??:0
@ 0x7f061f7aeb93 google::LogMessage::SendToLog() at ??:0
@ 0x7f061f7aa7cc google::LogMessage::Flush() at ??:0
@ 0x7f061f7abf59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f0620c14044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f0620bf8f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f0620bf6efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f06237a977a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f06237b2a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f062383376f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f06238474f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f0623832226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f0623841c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55a50191f75c std::function<>::operator()() at ??:0
@ 0x7f0620bfbe06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f0620bfc6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f0620bfe03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55a50191f75c std::function<>::operator()() at ??:0
@ 0x7f0620beb807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f06214296db start_thread at ??:0
@ 0x7f061e96171f clone at ??:0
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+---------+----------+----------------+-----------------
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | HEALTHY | <none> | 9 | 0
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | HEALTHY | <none> | 7 | 0
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | HEALTHY | <none> | 8 | 0
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 8 | 0
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | HEALTHY | <none> | 4 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 0 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 6 server(s) checked
local_ip_for_outbound_sockets | 127.9.215.193 | experimental | 127.9.215.193:34929
local_ip_for_outbound_sockets | 127.9.215.194 | experimental | 127.9.215.194:36971
local_ip_for_outbound_sockets | 127.9.215.195 | experimental | 127.9.215.195:33493
local_ip_for_outbound_sockets | 127.9.215.196 | experimental | 127.9.215.196:39655
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | all 6 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 6 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 6 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 6 server(s) checked
server_dump_info_format | pb | hidden | all 6 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.9.215.193:34929
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.9.215.194:36971
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.9.215.195:33493
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.9.215.196:39655
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | all 6 server(s) checked
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 6 server(s) checked
time_source | builtin | all 6 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 7 server(s) checked
Tablet Summary
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING [LEADER]
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): not running
State: STOPPED
Data state: TABLET_DATA_TOMBSTONED
Last status: No bootstrap required, opened a new log
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A B C | 1 | -1 | Yes
C | A* B C | 1 | -1 | Yes
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 63 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 62 | Yes
C | A B C* | 1 | -1 | Yes
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A* B C | 2 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A* B C | 2 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 61 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 62 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 66 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C | 2 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B C* D~ | 2 | 62 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A* B C | 2 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 54 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 841ade2081574e7f9e2b047a7f94550e
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* D~ | 1 | 61 | Yes
C | A B C* | 1 | -1 | Yes
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B C* | 2 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING [LEADER]
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): not running
State: STOPPED
Data state: TABLET_DATA_TOMBSTONED
Last status: Deleted tablet blocks from disk
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A B C | 1 | -1 | Yes
C | A* B C | 1 | -1 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+--------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | CONSENSUS_MISMATCH | 40 | 26 | 0 | 1 | 13
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 21
Third Quartile | 21
Maximum | 21
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 123
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
==================
Errors:
==================
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:45:40.966852 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | HEALTHY | <none> | 12 | 0
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | HEALTHY | <none> | 8 | 0
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | HEALTHY | <none> | 12 | 0
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 9 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 0 | 0
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+--------------------------------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
local_ip_for_outbound_sockets | 127.9.215.194 | experimental | 127.9.215.194:36971
local_ip_for_outbound_sockets | 127.9.215.195 | experimental | 127.9.215.195:33493
local_ip_for_outbound_sockets | 127.9.215.196 | experimental | 127.9.215.196:39655
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
server_dump_info_format | pb | hidden | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.9.215.194:36971
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.9.215.195:33493
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.9.215.196:39655
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+--------------------------------------------------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
time_source | builtin | 127.9.215.195:33493, 127.9.215.194:36971, 127.9.215.196:39655, and 2 other server(s)
Version Summary
Version | Servers
-----------------+-------------------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.194:36971, tserver@127.9.215.196:39655, and 3 other server(s)
Tablet Summary
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 5 | Yes
B | A* B C D~ | 1 | 5 | Yes
C | [config not available] | | |
D | A B C D~ | 1 | 5 | Yes
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 68 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 68 | Yes
D | A B C D~ | 1 | 68 | Yes
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 60 | Yes
C | A B* C D~ | 1 | 60 | Yes
D | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 66 | Yes
C | A B* C D~ | 1 | 66 | Yes
D | [config not available] | | |
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 64 | Yes
C | A B* C D~ | 1 | 64 | Yes
D | A B C D~ | 1 | 64 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 68 | Yes
C | A B C* D~ | 1 | 68 | Yes
D | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 62 | Yes
C | A B C* D~ | 1 | 62 | Yes
D | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 63 | Yes
C | A B C* D~ | 1 | 63 | Yes
D | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | -1 | Yes
C | A B* C | 4 | -1 | Yes
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 62 | Yes
C | A B* C D~ | 1 | 62 | Yes
D | A B C D~ | 1 | 62 | Yes
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 66 | Yes
C | A B C* D~ | 1 | 66 | Yes
D | A B C D~ | 1 | 66 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is recovering: 1 on-going tablet copies
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: INITIALIZED
Data state: TABLET_DATA_COPYING
Last status: Initialized. Waiting to start...
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 62 | Yes
C | A B C* D~ | 1 | 62 | Yes
D | A B C D~ | 1 | 62 | Yes
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 63 | Yes
C | A B* C D~ | 1 | 63 | Yes
D | A B C D~ | 1 | 63 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 65 | Yes
C | A B C* D~ | 1 | 65 | Yes
D | A B C D~ | 1 | 65 | Yes
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 70 | Yes
C | A B* C D~ | 1 | 70 | Yes
D | A B C D~ | 1 | 70 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): missing [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 60 | Yes
C | A B C* D~ | 1 | 60 | Yes
D | A B C D~ | 1 | 60 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 69 | Yes
C | A B C* D~ | 1 | 69 | Yes
D | A B C D~ | 1 | 69 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 59 | Yes
C | A B C* D~ | 1 | 59 | Yes
D | A B C D~ | 1 | 59 | Yes
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 64 | Yes
B | A* B C D~ | 1 | 64 | Yes
C | [config not available] | | |
D | A B C D~ | 1 | 64 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNDER_REPLICATED | 40 | 19 | 1 | 20 | 0
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 19
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 37
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | 37
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 141
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
F20240502 20:45:41.847172 12039 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1337ba25811549d7a26106cf190043aa.queue_time_us, tablet-download-pool-1337ba25811549d7a26106cf190043aa.queue_time_us) (tablet-download-pool-1337ba25811549d7a26106cf190043aa.run_cpu_time_us, tablet-download-pool-1337ba25811549d7a26106cf190043aa.run_cpu_time_us) (tablet-download-pool-1337ba25811549d7a26106cf190043aa.run_wall_time_us, tablet-download-pool-1337ba25811549d7a26106cf190043aa.run_wall_time_us) (tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.queue_time_us, tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.queue_time_us) (tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.run_cpu_time_us, tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.run_cpu_time_us) (tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.run_wall_time_us, tablet-download-pool-216a64422b93486ab47122f4cd02c5a8.run_wall_time_us) (tablet-download-pool-73639358398846b6b39ce031bb0978e6.queue_time_us, tablet-download-pool-73639358398846b6b39ce031bb0978e6.queue_time_us) (tablet-download-pool-73639358398846b6b39ce031bb0978e6.run_cpu_time_us, tablet-download-pool-73639358398846b6b39ce031bb0978e6.run_cpu_time_us) (tablet-download-pool-73639358398846b6b39ce031bb0978e6.run_wall_time_us, tablet-download-pool-73639358398846b6b39ce031bb0978e6.run_wall_time_us) (tablet-download-pool-886187ea81e7439680c5c8218046c865.queue_time_us, tablet-download-pool-886187ea81e7439680c5c8218046c865.queue_time_us) (tablet-download-pool-886187ea81e7439680c5c8218046c865.run_cpu_time_us, tablet-download-pool-886187ea81e7439680c5c8218046c865.run_cpu_time_us) (tablet-download-pool-886187ea81e7439680c5c8218046c865.run_wall_time_us, tablet-download-pool-886187ea81e7439680c5c8218046c865.run_wall_time_us) (tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.queue_time_us, tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.queue_time_us) (tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.run_cpu_time_us, tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.run_cpu_time_us) (tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.run_wall_time_us, tablet-download-pool-8a65cae70f43442e8431c324f8ed3d05.run_wall_time_us) (tablet-download-pool-92859d802c444ebfb61700ed9db22539.queue_time_us, tablet-download-pool-92859d802c444ebfb61700ed9db22539.queue_time_us) (tablet-download-pool-92859d802c444ebfb61700ed9db22539.run_cpu_time_us, tablet-download-pool-92859d802c444ebfb61700ed9db22539.run_cpu_time_us) (tablet-download-pool-92859d802c444ebfb61700ed9db22539.run_wall_time_us, tablet-download-pool-92859d802c444ebfb61700ed9db22539.run_wall_time_us) (tablet-download-pool-94afd17f64784e03bb5491b9521e0543.queue_time_us, tablet-download-pool-94afd17f64784e03bb5491b9521e0543.queue_time_us) (tablet-download-pool-94afd17f64784e03bb5491b9521e0543.run_cpu_time_us, tablet-download-pool-94afd17f64784e03bb5491b9521e0543.run_cpu_time_us) (tablet-download-pool-94afd17f64784e03bb5491b9521e0543.run_wall_time_us, tablet-download-pool-94afd17f64784e03bb5491b9521e0543.run_wall_time_us) (tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.queue_time_us, tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.queue_time_us) (tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.run_cpu_time_us, tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.run_cpu_time_us) (tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.run_wall_time_us, tablet-download-pool-9a5beec8c4f54f6c80b7aee36743be7a.run_wall_time_us) (tablet-download-pool-9b4b4658f56b415f8552562609484316.queue_time_us, tablet-download-pool-9b4b4658f56b415f8552562609484316.queue_time_us) (tablet-download-pool-9b4b4658f56b415f8552562609484316.run_cpu_time_us, tablet-download-pool-9b4b4658f56b415f8552562609484316.run_cpu_time_us) (tablet-download-pool-9b4b4658f56b415f8552562609484316.run_wall_time_us, tablet-download-pool-9b4b4658f56b415f8552562609484316.run_wall_time_us) (tablet-download-pool-b0b5919b4548446496882b5b3705bc66.queue_time_us, tablet-download-pool-b0b5919b4548446496882b5b3705bc66.queue_time_us) (tablet-download-pool-b0b5919b4548446496882b5b3705bc66.run_cpu_time_us, tablet-download-pool-b0b5919b4548446496882b5b3705bc66.run_cpu_time_us) (tablet-download-pool-b0b5919b4548446496882b5b3705bc66.run_wall_time_us, tablet-download-pool-b0b5919b4548446496882b5b3705bc66.run_wall_time_us) (tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.queue_time_us, tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.queue_time_us) (tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.run_cpu_time_us, tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.run_cpu_time_us) (tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.run_wall_time_us, tablet-download-pool-b2edc4a307c141ab8f3b40b90eec9202.run_wall_time_us) (tablet-download-pool-dbb41e3d17b14196b274546c485e13ec.queue_time_us, tablet-download-pool-dbb41e3d17b14196b274546c485e13ec.queue_time_us) (tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.queue_time_us, tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.queue_time_us) (tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.run_cpu_time_us, tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.run_cpu_time_us) (tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.run_wall_time_us, tablet-download-pool-e1abed559dea428e9718b5addbf72ca3.run_wall_time_us) (tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.queue_time_us, tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.queue_time_us) (tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.run_cpu_time_us, tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.run_cpu_time_us) (tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.run_wall_time_us, tablet-download-pool-e1dc93b317234f16bd5dd4b7dc2202cd.run_wall_time_us) (tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.queue_time_us, tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.queue_time_us) (tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.run_cpu_time_us, tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.run_cpu_time_us) (tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.run_wall_time_us, tablet-download-pool-e9c05a86c8604f2fa5111b2f210de66d.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fc1dca47dcd google::LogMessage::Fail() at ??:0
@ 0x7fc1dca4bb93 google::LogMessage::SendToLog() at ??:0
@ 0x7fc1dca477cc google::LogMessage::Flush() at ??:0
@ 0x7fc1dca48f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fc1ddeb1044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fc1dde95f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fc1dde93efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fc1e0a4677a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fc1e0a4fa45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fc1e0ad076f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fc1e0ae44f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fc1e0acf226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fc1e0adec0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x561e5a27f75c std::function<>::operator()() at ??:0
@ 0x7fc1dde98e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fc1dde996ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fc1dde9b03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x561e5a27f75c std::function<>::operator()() at ??:0
@ 0x7fc1dde88807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fc1de6c66db start_thread at ??:0
@ 0x7fc1dbbfe71f clone at ??:0
W20240502 20:45:42.269317 11024 connection.cc:535] client connection to 127.9.215.196:39655 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 20:45:42.269441 11024 meta_cache.cc:302] tablet 815c59e08a1b43498be81ad87e3a4caa: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 20:45:43.294900 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:44.383977 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:45.390465 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:46.399892 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:47.570271 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:48.649639 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:49.827862 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:50.850494 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:51.924590 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:53.087447 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:54.182883 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:55.440212 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:56.760324 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:57.790037 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:45:58.853909 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:45:59.964188 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:01.095252 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:02.267899 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:03.481922 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:04.726861 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:06.004055 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:07.331261 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:08.692608 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:10.080924 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:11.525135 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:12.988144 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:46:14.484877 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:15.506227 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:16.552521 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:17.599154 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:18.667822 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:19.758839 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:20.857873 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:21.973606 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:23.108623 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:24.249369 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:25.415400 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:26.606493 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:27.798352 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:29.004652 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:30.235801 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:31.480206 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:32.744231 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:34.022953 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:35.310560 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:36.619184 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:37.962257 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:39.304252 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:40.662969 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
F20240502 20:46:40.980823 16596 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.queue_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.queue_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_cpu_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_cpu_time_us) (tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_wall_time_us, tablet-download-pool-0dbf1515630f46249d47f1770c57aca2.run_wall_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.queue_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.queue_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_cpu_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_cpu_time_us) (tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_wall_time_us, tablet-download-pool-41c18355ea504c1d9b33ee57f34fea6e.run_wall_time_us) (tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.queue_time_us, tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.queue_time_us) (tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.run_cpu_time_us, tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.run_cpu_time_us) (tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.run_wall_time_us, tablet-download-pool-450adf30e7b648fdb56f2c21bee27272.run_wall_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.queue_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.queue_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_cpu_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_cpu_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_wall_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_wall_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.queue_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.queue_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_cpu_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_cpu_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_wall_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_wall_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.queue_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.queue_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_cpu_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_cpu_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_wall_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_wall_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.queue_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.queue_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_cpu_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_cpu_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_wall_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_wall_time_us) (tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us, tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fed27178dcd google::LogMessage::Fail() at ??:0
@ 0x7fed2717cb93 google::LogMessage::SendToLog() at ??:0
@ 0x7fed271787cc google::LogMessage::Flush() at ??:0
@ 0x7fed27179f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fed285e2044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fed285c6f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fed285c4efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fed2b17777a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fed2b180a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fed2b20176f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fed2b2154f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fed2b200226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fed2b20fc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55e40a99175c std::function<>::operator()() at ??:0
@ 0x7fed285c9e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fed285ca6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fed285cc03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55e40a99175c std::function<>::operator()() at ??:0
@ 0x7fed285b9807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fed28df76db start_thread at ??:0
@ 0x7fed2632f71f clone at ??:0
W20240502 20:46:42.029052 11026 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 41c18355ea504c1d9b33ee57f34fea6e after 341 attempt(s): Failed to write to server: (no server available): Write(tablet: 41c18355ea504c1d9b33ee57f34fea6e, num_ops: 3, num_attempts: 341) passed its deadline: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 20:46:42.032696 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:43.042562 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:44.049325 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:45.051910 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:46.075587 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:47.126116 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:48.129001 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:49.169741 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:50.189184 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:51.199617 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:52.208073 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:53.225087 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:46:54.247697 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:55.248178 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:56.281038 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:57.307220 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:58.320210 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:46:59.373556 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:00.376601 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:01.387434 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:02.396168 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:03.401329 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:47:04.445955 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:47:05.450281 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:06.478884 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:07.493438 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:08.505839 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:09.556385 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:10.612999 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:11.651008 11026 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:12.694159 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:47:13.722075 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:14.951977 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:15.987277 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:47:17.038084 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:18.105655 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:19.120672 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:20.132616 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:21.135849 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:22.178442 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:23.225389 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:24.240209 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:47:25.256070 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:26.333580 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:27.431800 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:28.444418 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:29.462795 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:30.497694 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:31.540256 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:32.585814 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:33.663130 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:34.685503 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:35.709657 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:36.926764 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:37.942054 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:38.944797 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:40.012239 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:41.037551 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:42.032563 11023 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 216a64422b93486ab47122f4cd02c5a8 after 1 attempt(s): Failed to write to server: 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): Write RPC to 127.9.215.195:33493 timed out after 59.997s (SENT)
F20240502 20:47:42.527788 18747 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.queue_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_cpu_time_us) (tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us, tablet-download-pool-13ce8f61951f4121aa3feb30989f6848.run_wall_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.queue_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.queue_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_cpu_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_cpu_time_us) (tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_wall_time_us, tablet-download-pool-2a589148b4594bdc9c712ce83239d966.run_wall_time_us) (tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.queue_time_us, tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.queue_time_us) (tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.run_cpu_time_us, tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.run_cpu_time_us) (tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.run_wall_time_us, tablet-download-pool-394cd3cfd7ab41538cd89da1efbbb23a.run_wall_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.queue_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.queue_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_cpu_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_cpu_time_us) (tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_wall_time_us, tablet-download-pool-49ede33794d448ec83c6ab75928bd782.run_wall_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.queue_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.queue_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_cpu_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_cpu_time_us) (tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_wall_time_us, tablet-download-pool-62fbccb3e8814fed9651808a003fcb94.run_wall_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.queue_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.queue_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_cpu_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_cpu_time_us) (tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_wall_time_us, tablet-download-pool-66137dda27c74f1a8ea8f0d2b73f1781.run_wall_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.queue_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_cpu_time_us) (tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us, tablet-download-pool-665f09edc2364cbf873f934d75927656.run_wall_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.queue_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_cpu_time_us) (tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us, tablet-download-pool-80f787f5fb2840e983b86a97ca0499ef.run_wall_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.queue_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_cpu_time_us) (tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us, tablet-download-pool-815c59e08a1b43498be81ad87e3a4caa.run_wall_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.queue_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_cpu_time_us) (tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us, tablet-download-pool-9f262ea8f7b44abcab7ec106bee6be5d.run_wall_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.queue_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_cpu_time_us) (tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us, tablet-download-pool-bf1027f4a5864d329145d8c612d4f9a1.run_wall_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.queue_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_cpu_time_us) (tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us, tablet-download-pool-c835a8c0f2784bd693650b0bc4fe0217.run_wall_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.queue_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.queue_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_cpu_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_cpu_time_us) (tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_wall_time_us, tablet-download-pool-c8d973222f314ea99471ded3590097fd.run_wall_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.queue_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.queue_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_cpu_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_cpu_time_us) (tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_wall_time_us, tablet-download-pool-d373b92bc8ad4b4f9d62d84e3e6417ab.run_wall_time_us) (tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us, tablet-download-pool-f97264b6cdfa4ec485bf9c3274f26ca7.queue_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fadb0659dcd google::LogMessage::Fail() at ??:0
@ 0x7fadb065db93 google::LogMessage::SendToLog() at ??:0
@ 0x7fadb06597cc google::LogMessage::Flush() at ??:0
@ 0x7fadb065af59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fadb1ac3044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fadb1aa7f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fadb1aa5efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fadb465877a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fadb4661a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fadb46e276f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | HEALTHY | <none> | 13 | 0
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 18 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+---------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.195 | experimental | 127.9.215.195:33493
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.9.215.195:33493
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+---------------------------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.195:33493, 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+-------------------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.197:35609, tserver@127.9.215.195:33493, and 1 other server(s)
Tablet Summary
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
@ 0x7fadb46f64f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A* B C | 1 | 124 | Yes
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 116 | Yes
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 1 | 123 | Yes
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [NONVOTER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 179 | Yes
C | A B C* D~ | 2 | 179 | Yes
D | A B C D~ | 2 | 179 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 1 | 121 | Yes
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): not running [NONVOTER]
State: BOOTSTRAPPING
Data state: TABLET_DATA_READY
Last status: Bootstrap starting.
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 173 | Yes
C | A B C* D~ | 2 | 173 | Yes
D | A B C D~ | 2 | 173 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 125 | Yes
D | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 118 | Yes
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 119 | Yes
D | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 125 | Yes
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [NONVOTER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 3 | 170 | Yes
C | A B C* D~ | 3 | 170 | Yes
D | A B C* D~ | 3 | 170 | Yes
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 123 | No
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
@ 0x7fadb46e1226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 119 | Yes
D | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 64 | No
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): missing [NONVOTER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
D = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 180 | Yes
C | A B C* D~ | 2 | 180 | Yes
D | [config not available] | | |
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 129 | Yes
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 179 | Yes
C | A B C* | 2 | 179 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 2 | 122 | No
D | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 121 | Yes
D | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 1 | 127 | Yes
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 1 | 116 | Yes
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 186 | Yes
C | A B C* | 2 | 186 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 125 | Yes
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 182 | Yes
C | A B C* | 2 | 182 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): RUNNING [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 115 | Yes
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 11 | 0 | 9 | 20
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 23
Maximum | 35
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 35
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 141
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
@ 0x7fadb46f0c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5610c882475c std::function<>::operator()() at ??:0
@ 0x7fadb1aaae06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fadb1aab6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fadb1aad03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5610c882475c std::function<>::operator()() at ??:0
@ 0x7fadb1a9a807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fadb22d86db start_thread at ??:0
@ 0x7fadaf81071f clone at ??:0
W20240502 20:47:42.767993 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:42.863703 11023 connection.cc:535] client connection to 127.9.215.195:33493 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 20:47:43.771023 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:44.780476 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:45.791374 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:46.797195 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:47.813500 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:47:48.813779 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:49.816870 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:47:50.818488 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:47:51.820122 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:52.828629 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:53.840260 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:54.841228 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:55.844405 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:47:56.852777 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:57.879117 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:47:58.885476 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:47:59.919656 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:00.933775 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:01.949681 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:02.962126 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:02.962126 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:03.972839 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:04.982702 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:06.009127 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:07.013757 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:08.030280 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:09.032908 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:10.042659 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:11.067616 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:12.079350 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:13.081920 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:14.096639 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:15.100908 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:16.104200 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:17.114531 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:18.130329 11023 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:19.157585 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:20.158018 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:21.164484 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:22.189024 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:23.303918 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:24.436295 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:25.583592 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:26.748517 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:27.757992 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:28.817164 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:29.887275 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:48:30.941012 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:31.942867 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:33.004493 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:34.056118 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:35.060149 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:36.063177 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:37.069322 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:38.084944 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:39.089928 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:40.103029 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:41.119704 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:42.149884 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:42.758347 11024 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet accdfb0117fa4d2eb5ebd06587827b95 after 342 attempt(s): Failed to write to server: (no server available): Write(tablet: accdfb0117fa4d2eb5ebd06587827b95, num_ops: 1, num_attempts: 342) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:43.149966 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:44.151592 11023 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:45.159724 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:46.159780 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:47.160033 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:48.161721 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:49.169718 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:50.170665 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:51.189554 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:52.205860 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:53.211661 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:48:54.230058 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:55.266701 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:56.299029 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:57.301461 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:48:58.318706 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:48:59.334038 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:00.336879 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:01.358057 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:02.405472 11026 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:03.405889 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:04.416842 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:05.462543 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:06.463799 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:07.544526 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:08.545991 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:09.572666 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:10.599545 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:11.686594 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:12.686800 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:13.741294 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:14.742410 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:15.749873 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:16.785638 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:17.833137 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:18.892391 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:19.897760 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:20.949210 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:21.949827 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:23.011514 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:24.044332 11023 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:25.045920 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:26.075415 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:27.076766 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:28.156407 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:29.164888 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:30.285673 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:31.314965 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:32.316795 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:33.414659 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:34.476511 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:35.484730 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:36.638619 11024 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:37.653597 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:38.739305 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:39.747445 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:40.761544 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:41.783702 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:42.814601 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:43.161743 11024 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet b0b5919b4548446496882b5b3705bc66 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: b0b5919b4548446496882b5b3705bc66, num_ops: 2, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.197:35609, tserver@127.9.215.198:39531
Tablet Summary
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:49:43.815815 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:44.826300 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:45.829178 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:46.838496 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:47.848719 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:48.850713 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:49.869783 11023 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:50.874269 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:51.874531 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:52.890967 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:49:53.893465 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:54.926872 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:49:55.940837 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:49:56.952342 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:57.958067 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:58.987481 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:49:59.990625 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:00.999866 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:02.003779 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:03.004523 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:04.006198 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:05.015736 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:06.038075 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:07.107832 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:08.108191 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:09.113772 11023 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:10.139585 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:11.173736 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:12.258162 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:13.332455 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:14.333607 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:15.335083 11026 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:16.339092 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:17.360149 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:18.404496 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:19.468881 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:20.547690 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:21.638664 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:22.639575 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:23.738258 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:24.742231 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:25.750423 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:26.770963 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:27.807562 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:28.846019 11023 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:29.915952 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:30.968346 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:32.067669 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:33.148412 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:34.158365 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:35.224995 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:36.354260 11023 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:37.364351 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:38.374907 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:50:39.375761 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:50:40.381924 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:50:41.399122 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:50:42.399345 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:43.420668 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:43.737744 11024 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 92859d802c444ebfb61700ed9db22539 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 92859d802c444ebfb61700ed9db22539, num_ops: 3, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:44.421382 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:45.422295 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:46.426009 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:47.435904 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:48.439285 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:49.442130 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:50.449223 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:51.485874 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:52.495333 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:50:53.510485 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:54.511310 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:55.540675 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:56.543081 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:50:57.543833 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:58.550575 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:50:59.566017 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:00.637274 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:01.668931 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:02.706041 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:03.707162 11023 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:04.711186 11023 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:05.719558 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:06.772984 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:51:07.773154 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:08.782508 11023 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:09.869266 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:10.919315 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:11.958432 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:12.967228 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:14.029844 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:15.118240 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:16.126451 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:17.153133 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:18.190621 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:19.249352 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:20.324334 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:21.412197 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:51:22.412559 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:23.505558 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:24.519495 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:25.523264 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:26.531447 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:27.542358 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:28.561903 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:29.633875 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:30.682763 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:31.778298 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:32.843374 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:33.963012 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:35.064271 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:36.080710 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:37.174116 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:38.193019 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:39.301723 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:40.302284 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:41.311527 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:42.327674 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:43.355396 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:43.956159 11024 meta_cache.cc:1015] Timed out: LookupRpc { table: 'RemoveReplaceInCycle', partition-key: (RANGE (key): 107374182), attempt: 1 } failed: LookupRpc timed out after deadline expired: GetTableLocations RPC to 127.9.215.254:38205 timed out after 0.000s (SENT)
W20240502 20:51:43.956296 11024 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 886187ea81e7439680c5c8218046c865 after 343 attempt(s): LookupRpc { table: 'RemoveReplaceInCycle', partition-key: (RANGE (key): 107374182), attempt: 1 } failed: LookupRpc timed out after deadline expired: GetTableLocations RPC to 127.9.215.254:38205 timed out after 0.000s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.197:35609, tserver@127.9.215.198:39531
Tablet Summary
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:51:44.507591 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:45.508117 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:46.510424 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:47.515964 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:48.517714 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:49.528282 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:50.528321 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:51.531363 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:52.532045 11026 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:53.565373 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:54.565418 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:55.565766 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:56.616667 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:51:57.663403 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:51:58.722137 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:51:59.747128 11026 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:00.754067 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:01.756665 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:02.780977 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:03.830688 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:04.831631 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:05.846670 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:06.892089 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:07.962000 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:08.962728 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:09.965786 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:52:10.995525 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:12.028578 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:13.102840 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:14.176522 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:15.179651 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:16.186251 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:17.191705 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:18.192646 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:19.240474 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:20.303962 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:21.312017 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:22.375239 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:23.478806 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:24.481870 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:25.500416 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:26.616501 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:27.617044 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:28.668051 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:29.682466 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:30.773357 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:31.814813 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:32.926179 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:33.995791 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:35.131618 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:36.141973 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:37.186300 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:38.190140 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:52:39.209174 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:40.210837 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:41.215541 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:42.222193 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:43.222849 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:44.224954 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:44.568884 11026 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet dbb41e3d17b14196b274546c485e13ec after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: dbb41e3d17b14196b274546c485e13ec, num_ops: 3, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:45.228955 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:52:46.238474 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:47.246068 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:48.271698 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:49.272974 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:50.289520 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:51.294612 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:52.298945 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:53.300905 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:54.304859 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:52:55.313792 11023 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:56.320189 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:57.324708 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:52:58.338838 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:52:59.348831 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:00.357048 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:01.357609 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:02.358753 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:03.375486 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:04.401548 11026 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:05.403298 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:06.420425 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:07.429950 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:08.444027 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:09.447584 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:10.447937 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:11.458176 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:53:12.492887 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:13.493131 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:14.499580 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:15.521638 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:16.536492 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:17.552701 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:18.557082 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:19.575181 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:20.576653 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:21.610075 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:22.615226 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:23.621356 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:24.649133 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:25.649385 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:26.709558 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:27.729287 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:28.761072 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:29.763569 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:30.787761 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:31.798331 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:32.813468 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:33.824273 11026 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:34.898135 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:35.941123 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:36.955329 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:37.969147 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:39.009127 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:40.087271 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:41.090780 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:42.100924 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:43.124321 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:44.151485 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:44.851614 11026 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 216a64422b93486ab47122f4cd02c5a8 after 343 attempt(s): Failed to write to server: (no server available): Write(tablet: 216a64422b93486ab47122f4cd02c5a8, num_ops: 1, num_attempts: 343) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.198:39531, tserver@127.9.215.197:35609
Tablet Summary
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:53:45.435341 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:46.438014 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:47.444569 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:48.445099 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:49.447080 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:50.450400 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:51.455077 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:52.461205 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:53.466007 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:54.484301 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:53:55.489573 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:56.494900 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:53:57.496829 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:58.500638 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:53:59.506460 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:00.507045 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:01.514983 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:02.516649 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:03.522711 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:04.527180 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:05.553799 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:06.571735 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:07.578218 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:08.590591 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:09.592001 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:10.600136 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:11.605393 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:12.610394 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:13.621596 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:14.626291 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:15.626713 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:16.645109 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:17.647513 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:18.650558 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:19.658757 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:20.663121 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:21.663172 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:22.667160 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:23.696799 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:24.719139 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:25.727064 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:26.738098 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:27.740008 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:28.747344 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:29.749698 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:30.785140 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:31.835081 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:32.915199 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:33.924925 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:34.937590 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:35.953967 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:37.002103 11026 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:38.029652 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:39.030292 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:40.099159 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:41.109553 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:42.113972 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:43.117502 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:44.135967 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:45.168346 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:45.512861 11026 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 73639358398846b6b39ce031bb0978e6 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 73639358398846b6b39ce031bb0978e6, num_ops: 3, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:46.172317 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:47.174046 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:48.177249 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:49.184518 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:50.189873 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:51.191313 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:52.220888 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:53.228988 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:54.230064 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:55.232915 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:56.239732 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:54:57.254949 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:54:58.278515 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:54:59.307691 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:00.314783 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:01.362052 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:02.384138 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:03.385869 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:04.392478 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:05.405133 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:06.408370 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:07.436491 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:08.441707 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:09.442378 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:10.450965 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:11.452649 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:12.454547 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:13.523643 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:14.524488 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:15.548177 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:16.577041 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:17.577244 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:18.596416 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:19.630426 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:20.642604 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:21.643201 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:22.651093 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:23.744113 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:24.750190 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:25.774760 11026 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:26.843664 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:27.886233 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:28.888847 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:29.894749 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:30.945647 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:31.957690 11026 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:32.965129 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:34.006628 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:35.020546 11023 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:36.033506 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:37.049564 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:38.135262 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:39.141430 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:40.169404 11024 meta_cache.cc:302] tablet accdfb0117fa4d2eb5ebd06587827b95: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:41.169885 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:42.171243 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:43.181047 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:44.201747 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:45.228328 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:45.849721 11024 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 8a65cae70f43442e8431c324f8ed3d05 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 8a65cae70f43442e8431c324f8ed3d05, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.198:39531, tserver@127.9.215.197:35609
Tablet Summary
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:55:46.295011 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:47.306996 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:48.313153 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:49.327328 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:50.339175 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:51.349730 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:52.349813 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:55:53.353539 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:54.355291 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:55.377955 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:56.386147 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:57.390111 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:55:58.399752 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:55:59.403223 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:00.403357 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:01.414278 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:02.421325 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:03.430086 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:04.462525 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:05.472461 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:56:06.474442 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:07.476574 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:08.492377 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:09.559552 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:10.570396 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:11.572769 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:12.597851 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:13.603718 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:14.610136 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:15.614941 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:16.640202 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:17.713330 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:18.722381 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:19.749513 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:20.789960 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:21.844612 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:22.925130 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:23.931018 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:24.980563 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:25.991860 11026 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:27.009022 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:28.013866 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:56:29.027570 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:30.037341 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:31.043298 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:32.111904 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:33.164469 11024 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:34.262238 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:35.278105 11024 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:36.281824 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:37.288020 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:38.375202 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:39.378682 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:40.408104 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:41.465353 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:42.469144 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:43.476846 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:44.493728 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:45.519358 11023 meta_cache.cc:302] tablet 92859d802c444ebfb61700ed9db22539: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:46.403165 11023 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet e9c05a86c8604f2fa5111b2f210de66d after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: e9c05a86c8604f2fa5111b2f210de66d, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:46.621176 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:47.621280 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:48.639770 11023 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:49.645048 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:50.648262 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:51.690366 11023 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:52.691205 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:53.696328 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:54.704454 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:56:55.705706 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:56.710804 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:57.724828 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:56:58.747351 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:56:59.748873 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:00.765246 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:01.794080 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:02.802995 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:03.854396 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:04.862130 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:05.871491 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:06.872707 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:07.882145 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:08.883337 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:09.899538 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:10.962333 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:11.984227 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:12.992051 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:14.002050 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:15.006021 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:16.062052 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:17.066983 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:18.067422 11026 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:19.072896 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:20.078087 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:21.116583 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:22.174044 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:23.250286 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:24.266366 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:25.324551 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:26.329121 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:27.333326 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:28.366631 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:29.449276 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:30.499373 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:31.522862 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:32.598271 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:33.647022 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:34.649645 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:35.678936 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:36.771960 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:37.792824 11023 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:38.862455 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:39.872587 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:40.970201 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:41.972510 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:42.975548 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:43.979856 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:44.998292 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:46.030632 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:46.719825 11023 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 1337ba25811549d7a26106cf190043aa after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 1337ba25811549d7a26106cf190043aa, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.198:39531, tserver@127.9.215.197:35609
Tablet Summary
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:57:47.204864 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:48.211750 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:57:49.214584 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:50.224290 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:51.229586 11026 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:52.251422 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:53.254720 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:57:54.274041 11024 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:55.282619 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:56.287272 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:57.298573 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:58.305083 11024 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:57:59.311342 11023 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:00.318372 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:01.319778 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:02.329416 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:03.337575 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:04.358846 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:05.382733 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:06.386485 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:07.391057 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:08.397280 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:09.428314 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:10.459391 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:11.549070 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:12.561494 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:13.579113 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:14.583876 11023 meta_cache.cc:302] tablet dbb41e3d17b14196b274546c485e13ec: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:15.589509 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:16.652395 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:17.658246 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:18.665055 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:19.665854 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:20.678606 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:21.711807 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:22.771704 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:23.843876 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:24.934366 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:26.041545 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:27.048362 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:28.050598 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 564db356872644cca9367376ee209e1f (127.9.215.196:39655) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111)
W20240502 20:58:29.173709 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:30.190282 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:31.227118 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:32.268826 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:33.331418 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:34.392557 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:35.479308 11026 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:36.480266 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:37.516947 11023 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:38.641085 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:39.641919 11024 meta_cache.cc:302] tablet 41c18355ea504c1d9b33ee57f34fea6e: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:40.664198 11026 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:41.728772 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:42.743144 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:43.747258 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:44.762084 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:45.782640 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:46.796815 11026 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:47.310847 11023 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 73639358398846b6b39ce031bb0978e6 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 73639358398846b6b39ce031bb0978e6, num_ops: 2, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:47.798666 11026 meta_cache.cc:302] tablet aba3fe0e75834251b819c7fdf335fe5a: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:48.799139 11024 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:49.807505 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:50.811416 11026 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:51.826232 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:52.836297 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:53.850235 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:54.850369 11024 meta_cache.cc:302] tablet 9b4b4658f56b415f8552562609484316: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:55.851059 11026 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:56.851266 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:58:57.856346 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:58:58.857400 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:58:59.859026 11024 meta_cache.cc:302] tablet 58164cd9de50468f96d91a032be20593: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:00.874505 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:01.879684 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:02.908674 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:03.926874 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:04.961645 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:05.978927 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:06.994912 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:08.007750 11024 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:09.020140 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:10.030567 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:11.032089 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:12.035768 11026 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:13.038064 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:14.062737 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:15.092485 11024 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:16.120647 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:17.142477 11023 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:18.147483 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:19.192698 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:20.203591 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:21.231810 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:22.280393 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:23.290601 11024 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:24.349817 11024 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:25.382382 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:26.390452 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:27.399147 11023 meta_cache.cc:302] tablet e9c05a86c8604f2fa5111b2f210de66d: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:28.467841 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:29.482769 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:30.488106 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:31.542740 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:32.575134 11024 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:33.641357 11023 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:34.704067 11023 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:35.711345 11023 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:36.728670 11024 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:37.739532 11023 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:38.785598 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:39.794543 11026 meta_cache.cc:302] tablet b2edc4a307c141ab8f3b40b90eec9202: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:40.804821 11026 meta_cache.cc:302] tablet e1abed559dea428e9718b5addbf72ca3: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:41.879088 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:42.894722 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:43.897753 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:44.912425 11026 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:45.932804 11026 meta_cache.cc:302] tablet 8a65cae70f43442e8431c324f8ed3d05: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:46.938282 11026 meta_cache.cc:302] tablet 1337ba25811549d7a26106cf190043aa: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:47.628526 11023 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 1337ba25811549d7a26106cf190043aa after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 1337ba25811549d7a26106cf190043aa, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
9290ae293c334b76b6e192a466b594b8 | 127.9.215.254:38205 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.9.215.212:38961 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
749a9092579845c28b8b65b24a5520c8 | 127.9.215.198:39531 | HEALTHY | <none> | 19 | 0
cc70668e94e44fe9b3f7ea316962c766 | 127.9.215.197:35609 | HEALTHY | <none> | 2 | 0
168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | UNAVAILABLE | <none> | n/a | n/a
1bde2b26ef9b45e38c446a1a41d7c0de | 127.9.215.194:36971 | UNAVAILABLE | <none> | n/a | n/a
564db356872644cca9367376ee209e1f | 127.9.215.196:39655 | UNAVAILABLE | <none> | n/a | n/a
841ade2081574e7f9e2b047a7f94550e | 127.9.215.193:34929 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.9.215.195:33493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.194:36971: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.196:39655: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.196:39655: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.9.215.193:34929: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.9.215.198:39531, 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.197 | experimental | 127.9.215.197:35609
local_ip_for_outbound_sockets | 127.9.215.198 | experimental | 127.9.215.198:39531
never_fsync | true | unsafe,advanced | 127.9.215.198:39531, 127.9.215.197:35609
openssl_security_level_override | 0 | unsafe,hidden | 127.9.215.198:39531, 127.9.215.197:35609
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.9.215.198:39531, 127.9.215.197:35609
rpc_server_allow_ephemeral_ports | true | unsafe | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_format | pb | hidden | 127.9.215.198:39531, 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.9.215.197:35609
server_dump_info_path | /tmp/dist-test-taskZRhIiY/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714682727856035-10079-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.9.215.198:39531
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.9.215.198:39531, 127.9.215.197:35609
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.9.215.212:38961 | 127.9.215.198:39531, 127.9.215.197:35609
time_source | builtin | 127.9.215.198:39531, 127.9.215.197:35609
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.9.215.254:38205, tserver@127.9.215.197:35609, tserver@127.9.215.198:39531
Tablet Summary
Tablet 665f09edc2364cbf873f934d75927656 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
All reported replicas are:
A = 749a9092579845c28b8b65b24a5520c8
B = 168ea3788aeb40419f31ec54a55d9ba6
C = cc70668e94e44fe9b3f7ea316962c766
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 14 | Yes
B | [config not available] | | |
C | A* B C | 2 | 14 | Yes
Tablet b2edc4a307c141ab8f3b40b90eec9202 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 1bde2b26ef9b45e38c446a1a41d7c0de
B = 841ade2081574e7f9e2b047a7f94550e
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 394cd3cfd7ab41538cd89da1efbbb23a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 886187ea81e7439680c5c8218046c865 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 13ce8f61951f4121aa3feb30989f6848 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 127 | Yes
B | A B* C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 94afd17f64784e03bb5491b9521e0543 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet bf1027f4a5864d329145d8c612d4f9a1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 1337ba25811549d7a26106cf190043aa of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 80f787f5fb2840e983b86a97ca0499ef of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 230 | Yes
C | A B C* | 2 | 230 | Yes
Tablet 67bc1867f7c04fed8f3198a8e4b842a4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c835a8c0f2784bd693650b0bc4fe0217 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 118 | Yes
B | A* B C | 2 | 118 | Yes
C | [config not available] | | |
Tablet e1abed559dea428e9718b5addbf72ca3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d373b92bc8ad4b4f9d62d84e3e6417ab of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 125 | Yes
B | A* B C | 3 | 125 | Yes
C | [config not available] | | |
Tablet dbb41e3d17b14196b274546c485e13ec of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 815c59e08a1b43498be81ad87e3a4caa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 124 | Yes
B | A B* C | 2 | 124 | Yes
C | [config not available] | | |
Tablet accdfb0117fa4d2eb5ebd06587827b95 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 49ede33794d448ec83c6ab75928bd782 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 172 | Yes
B | A B* C | 3 | 172 | Yes
C | [config not available] | | |
Tablet 9a5beec8c4f54f6c80b7aee36743be7a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 168ea3788aeb40419f31ec54a55d9ba6
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 73 | Yes
C | A B C* | 3 | 73 | Yes
Tablet c8d973222f314ea99471ded3590097fd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 181 | Yes
B | A B* C | 1 | 181 | Yes
C | [config not available] | | |
Tablet 9b4b4658f56b415f8552562609484316 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 439d311e99ff408aa2b8fee445449d6c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 239 | Yes
C | A B C* | 1 | 239 | Yes
Tablet 216a64422b93486ab47122f4cd02c5a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 9f262ea8f7b44abcab7ec106bee6be5d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 129 | Yes
B | A B* C | 1 | 129 | Yes
C | [config not available] | | |
Tablet 734dd3b3e0844a159d1299c4a1f821e1 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 62fbccb3e8814fed9651808a003fcb94 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 4 | 183 | Yes
B | A B* C | 4 | 183 | Yes
C | [config not available] | | |
Tablet e9c05a86c8604f2fa5111b2f210de66d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet f97264b6cdfa4ec485bf9c3274f26ca7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 238 | Yes
C | A B C* | 2 | 238 | Yes
Tablet aba3fe0e75834251b819c7fdf335fe5a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 450adf30e7b648fdb56f2c21bee27272 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 237 | Yes
C | A B C* | 2 | 237 | Yes
Tablet 58164cd9de50468f96d91a032be20593 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 66137dda27c74f1a8ea8f0d2b73f1781 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 128 | Yes
B | A B* C | 3 | 128 | Yes
C | [config not available] | | |
Tablet e1dc93b317234f16bd5dd4b7dc2202cd of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2a589148b4594bdc9c712ce83239d966 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
All reported replicas are:
A = cc70668e94e44fe9b3f7ea316962c766
B = 749a9092579845c28b8b65b24a5520c8
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 174 | Yes
B | A B* C | 1 | 174 | Yes
C | [config not available] | | |
Tablet 92859d802c444ebfb61700ed9db22539 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable [LEADER]
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 5d2d226d2dd74c12aa21d7f2058ecec5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet b0b5919b4548446496882b5b3705bc66 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [NONVOTER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
D = 564db356872644cca9367376ee209e1f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c6d94a7282264566a4cd3dd57021cf3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 246 | Yes
C | A B C* | 2 | 246 | Yes
Tablet 73639358398846b6b39ce031bb0978e6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0dbf1515630f46249d47f1770c57aca2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable
cc70668e94e44fe9b3f7ea316962c766 (127.9.215.197:35609): RUNNING
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING [LEADER]
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = cc70668e94e44fe9b3f7ea316962c766
C = 749a9092579845c28b8b65b24a5520c8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 241 | Yes
C | A B C* | 2 | 241 | Yes
Tablet 8a65cae70f43442e8431c324f8ed3d05 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493): TS unavailable [LEADER]
All reported replicas are:
A = 841ade2081574e7f9e2b047a7f94550e
B = 1bde2b26ef9b45e38c446a1a41d7c0de
C = 168ea3788aeb40419f31ec54a55d9ba6
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 41c18355ea504c1d9b33ee57f34fea6e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
564db356872644cca9367376ee209e1f (127.9.215.196:39655): TS unavailable [LEADER]
749a9092579845c28b8b65b24a5520c8 (127.9.215.198:39531): RUNNING
841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929): TS unavailable
1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971): TS unavailable
All reported replicas are:
A = 564db356872644cca9367376ee209e1f
B = 749a9092579845c28b8b65b24a5520c8
C = 841ade2081574e7f9e2b047a7f94550e
D = 1bde2b26ef9b45e38c446a1a41d7c0de
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D | | | Yes
A | [config not available] | | |
B | A* B C D | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 20 | 20
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 22
Third Quartile | 22
Maximum | 32
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
------+----------------------------------+---------------------+---------------
Big | 168ea3788aeb40419f31ec54a55d9ba6 | 127.9.215.195:33493 | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 20:59:48.105628 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:49.111413 11026 meta_cache.cc:302] tablet b0b5919b4548446496882b5b3705bc66: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:50.129801 11024 meta_cache.cc:302] tablet 67bc1867f7c04fed8f3198a8e4b842a4: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:51.135005 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:52.136277 11026 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 841ade2081574e7f9e2b047a7f94550e (127.9.215.193:34929) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.193:34929: connect: Connection refused (error 111)
W20240502 20:59:53.154970 11024 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:54.161307 11024 meta_cache.cc:302] tablet 94afd17f64784e03bb5491b9521e0543: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
W20240502 20:59:55.164681 11023 meta_cache.cc:302] tablet 734dd3b3e0844a159d1299c4a1f821e1: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:56.169687 11023 meta_cache.cc:302] tablet 886187ea81e7439680c5c8218046c865: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
W20240502 20:59:57.171878 11023 meta_cache.cc:302] tablet 73639358398846b6b39ce031bb0978e6: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
************************ BEGIN STACKS **************************
W20240502 20:59:58.176676 11024 meta_cache.cc:302] tablet 216a64422b93486ab47122f4cd02c5a8: replica 1bde2b26ef9b45e38c446a1a41d7c0de (127.9.215.194:36971) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.194:36971: connect: Connection refused (error 111)
[New LWP 10082]
[New LWP 10083]
[New LWP 10084]
[New LWP 10085]
[New LWP 10086]
[New LWP 10087]
[New LWP 10088]
[New LWP 10992]
[New LWP 10993]
[New LWP 10994]
[New LWP 10995]
[New LWP 11023]
[New LWP 11024]
[New LWP 11026]
[New LWP 11027]
[New LWP 11543]
[New LWP 11544]
[New LWP 23115]
[New LWP 23183]
[New LWP 23185]
[New LWP 23186]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7f59896dd380 (LWP 10079) "raft_consensus_" 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7f59876b8700 (LWP 10082) "raft_consensus_" 0x00007f5995d9732a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7f5985cdb700 (LWP 10083) "kernel-watcher-" 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7f59854da700 (LWP 10084) "rpc reactor-100" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7f5984cd9700 (LWP 10085) "rpc reactor-100" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7f59844d8700 (LWP 10086) "rpc reactor-100" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7f5983cd7700 (LWP 10087) "rpc reactor-100" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7f59834d6700 (LWP 10088) "rpc reactor-100" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7f5982cd5700 (LWP 10992) "rpc reactor-109" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7f59822cd700 (LWP 10993) "rpc reactor-109" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7f5981acc700 (LWP 10994) "rpc reactor-109" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7f59812cb700 (LWP 10995) "rpc reactor-109" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7f597fac8700 (LWP 11023) "rpc reactor-110" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7f597f2c7700 (LWP 11024) "rpc reactor-110" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7f597eac6700 (LWP 11026) "rpc reactor-110" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7f597e2c5700 (LWP 11027) "rpc reactor-110" 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7f597dac4700 (LWP 11543) "raft_consensus_" 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
18 Thread 0x7f597d2c3700 (LWP 11544) "raft_consensus_" 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
19 Thread 0x7f5980aca700 (LWP 23115) "raft_consensus_" 0x00007f5995d92ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
20 Thread 0x7f597b2bf700 (LWP 23183) "client-negotiat" 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
21 Thread 0x7f597c2c1700 (LWP 23185) "client-negotiat" 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
22 Thread 0x7f59802c9700 (LWP 23186) "client-negotiat" 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 22 (Thread 0x7f59802c9700 (LWP 23186)):
#0 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f59958519d4 in kudu::ConditionVariable::WaitFor (this=0x7f59802c8690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007f599599f8d1 in kudu::ThreadPool::DispatchThread (this=0x5650d2be0f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007f59959a06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x5650d2fa1558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007f59959a203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d2fa1558) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d2fa1500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 21 (Thread 0x7f597c2c1700 (LWP 23185)):
#0 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f59958519d4 in kudu::ConditionVariable::WaitFor (this=0x7f597c2c0690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007f599599f8d1 in kudu::ThreadPool::DispatchThread (this=0x5650d2a8ef00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007f59959a06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x5650d2eba558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007f59959a203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d2eba558) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d2eba500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 20 (Thread 0x7f597b2bf700 (LWP 23183)):
#0 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f59958519d4 in kudu::ConditionVariable::WaitFor (this=0x7f597b2be690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007f599599f8d1 in kudu::ThreadPool::DispatchThread (this=0x5650d2be0f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007f59959a06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x5650d2ebb258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007f59959a203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d2ebb258) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d2ebb200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 19 (Thread 0x7f5980aca700 (LWP 23115)):
#0 0x00007f5995d92ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f5995851665 in kudu::ConditionVariable::Wait (this=0x5650d2c1f3b0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:55
#2 0x00007f59965db4c1 in kudu::client::KuduSession::Data::Flush (this=0x5650d2c1f320) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/session-internal.cc:257
#3 0x00007f59964e5638 in kudu::client::KuduSession::Flush (this=0x5650d2cf7d60) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/client.cc:1524
#4 0x00007f5996e0c7e3 in kudu::TestWorkload::WriteThread (this=0x7ffcd4582bc0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/test_workload.cc:204
#5 0x00007f5996e16b8f in std::__invoke_impl<void, void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__f=@0x5650d2f99fd0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7f5996e0bd88 <kudu::TestWorkload::WriteThread()>, __t=@0x5650d2f99fc8: 0x7ffcd4582bc0) at /usr/include/c++/7/bits/invoke.h:73
#6 0x00007f5996e16436 in std::__invoke<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__fn=@0x5650d2f99fd0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7f5996e0bd88 <kudu::TestWorkload::WriteThread()>) at /usr/include/c++/7/bits/invoke.h:95
#7 0x00007f5996e17b07 in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::_M_invoke<0ul, 1ul> (this=0x5650d2f99fc8) at /usr/include/c++/7/thread:234
#8 0x00007f5996e17a6d in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::operator() (this=0x5650d2f99fc8) at /usr/include/c++/7/thread:243
#9 0x00007f5996e179e2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> > >::_M_run (this=0x5650d2f99fc0) at /usr/include/c++/7/thread:186
#10 0x00007f599495a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 18 (Thread 0x7f597d2c3700 (LWP 11544)):
#0 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f5994f19282 in base::SleepForNanoseconds (nanoseconds=1000000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f599592fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x00005650d13c59cc in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x5650d2bffaa8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:215
#4 0x00005650d13c8299 in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x00005650d13c7ef9 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x00005650d13c8d36 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x5650d2bffaa8) at /usr/include/c++/7/thread:234
#7 0x00005650d13c8ce3 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x5650d2bffaa8) at /usr/include/c++/7/thread:243
#8 0x00005650d13c8ca2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x5650d2bffaa0) at /usr/include/c++/7/thread:186
#9 0x00007f599495a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 17 (Thread 0x7f597dac4700 (LWP 11543)):
#0 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f5994f19282 in base::SleepForNanoseconds (nanoseconds=250000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f599592fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x00005650d13c552c in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x5650d2bffa78) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:196
#4 0x00005650d13c812f in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x00005650d13c7df6 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x00005650d13c8d62 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x5650d2bffa78) at /usr/include/c++/7/thread:234
#7 0x00005650d13c8d07 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x5650d2bffa78) at /usr/include/c++/7/thread:243
#8 0x00005650d13c8cc2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x5650d2bffa70) at /usr/include/c++/7/thread:186
#9 0x00007f599495a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7f597e2c5700 (LWP 11027)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b82880, timeout=0.099711267353995936) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b82880, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae3560, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae3558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d2c78658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d2c78658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d2c78600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7f597eac6700 (LWP 11026)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b82400, timeout=0.041841522353934124) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b82400, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae33a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae3398) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d2c78558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d2c78558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d2c78500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7f597f2c7700 (LWP 11024)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b81f80, timeout=0.042269583354936913) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b81f80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae31e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae31d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29eee58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29eee58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29eee00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7f597fac8700 (LWP 11023)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2bf0d80, timeout=0.047221182354405755) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2bf0d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae3020, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae3018) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29eef58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29eef58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29eef00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7f59812cb700 (LWP 10995)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b81b00, timeout=0.099459708351787413) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b81b00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2e60, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ee358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ee358) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ee300) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7f5981acc700 (LWP 10994)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b81680, timeout=0.099451887352188351) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b81680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2ca0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2c98) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ee958) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ee958) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ee900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7f59822cd700 (LWP 10993)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2bf1680, timeout=0.098930708350962959) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2bf1680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2ae0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2ad8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ee558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ee558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ee500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7f5982cd5700 (LWP 10992)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2bf1200, timeout=0.099399359351082239) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2bf1200, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2920, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2918) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ee758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ee758) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ee700) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7f59834d6700 (LWP 10088)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b80d80, timeout=0.099533185330074048) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b80d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2220, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2218) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ef458) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ef458) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ef400) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7f5983cd7700 (LWP 10087)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b80900, timeout=0.099949575331265805) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b80900, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae23e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae23d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ef058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ef058) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ef000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7f59844d8700 (LWP 10086)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b80480, timeout=0.099969541330210632) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b80480, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2760, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ee858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ee858) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ee800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7f5984cd9700 (LWP 10085)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2b80000, timeout=0.099903150330646895) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2b80000, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae2060, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29ef258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ef258) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ef200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7f59854da700 (LWP 10084)):
#0 0x00007f5994017a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f598f81ade0 in epoll_poll (loop=0x5650d2a43600, timeout=0.099579107332829153) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f598f81e49e in ev_run (loop=0x5650d2a43600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f5995988ba3 in ev::loop_ref::run (this=0x5650d2ae25a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f599219d31d in kudu::rpc::ReactorThread::RunThread (this=0x5650d2ae2598) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f5992199c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x5650d29eec58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f59921a1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29eec58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29eec00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7f5985cdb700 (LWP 10083)):
#0 0x00007f5995d92fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f59958517eb in kudu::ConditionVariable::WaitUntil (this=0x5650d29d72d8, until=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:87
#2 0x00007f5996929fde in kudu::CountDownLatch::WaitUntil (this=0x5650d29d72a0, when=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:87
#3 0x00007f599692a088 in kudu::CountDownLatch::WaitFor (this=0x5650d29d72a0, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:97
#4 0x00007f59958e9166 in kudu::KernelStackWatchdog::RunThread (this=0x5650d29d7200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007f59958e8375 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x5650d29ef158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007f59958e9aca in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f5996c1c856 in std::function<void ()>::operator()() const (this=0x5650d29ef158) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f599598f807 in kudu::Thread::SuperviseThread (arg=0x5650d29ef100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7f59876b8700 (LWP 10082)):
#0 0x00007f5995d9732a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f59959867b5 in kudu::Subprocess::DoWait (this=0x7f59876b7380, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:817
#2 0x00007f5995983be8 in kudu::Subprocess::Wait (this=0x7f59876b7380, wait_status=0x0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:544
#3 0x00007f59958d6b7c in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:258
#4 0x00007f59958d66ab in kudu::PstackWatcher::RunGdbStackDump (pid=10079, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:218
#5 0x00007f59958d5f09 in kudu::PstackWatcher::DumpPidStacks (pid=10079, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:169
#6 0x00007f59958d5e82 in kudu::PstackWatcher::DumpStacks (flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:162
#7 0x00007f5996cc7087 in kudu::<lambda()>::operator()(void) const (__closure=0x5650d2a559f8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:60
#8 0x00007f5996cc7888 in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#9 0x00007f5996cc75dc in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#10 0x00007f5996cc7e6e in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x5650d2a559f8) at /usr/include/c++/7/thread:234
#11 0x00007f5996cc7e13 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x5650d2a559f8) at /usr/include/c++/7/thread:243
#12 0x00007f5996cc7dce in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x5650d2a559f0) at /usr/include/c++/7/thread:186
#13 0x00007f599495a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007f5995d8c6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007f599401771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7f59896dd380 (LWP 10079)):
#0 0x00007f5995d96d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f5994f19282 in base::SleepForNanoseconds (nanoseconds=1000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f599592fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x00005650d13c68e1 in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody (this=0x5650d2a43a80) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:229
#4 0x00007f5995b320ed in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7f5995b3382b "the test body", method=<optimized out>, object=0x5650d2a43a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#5 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x5650d2a43a80, method=<optimized out>, location=location@entry=0x7f5995b3382b "the test body") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#6 0x00007f5995b26bdb in testing::Test::Run (this=0x5650d2a43a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#7 0x00007f5995b26d9d in testing::TestInfo::Run (this=0x5650d29d6240) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#8 0x00007f5995b27377 in testing::TestSuite::Run (this=0x5650d29d6480) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#9 0x00007f5995b2777c in testing::internal::UnitTestImpl::RunAllTests (this=<optimized out>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870
#10 0x00007f5995b3260d in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7f5995b353d8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x5650d2a8e280) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#11 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x5650d2a8e280, method=<optimized out>, location=location@entry=0x7f5995b353d8 "auxiliary test code (environments or event listeners)") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#12 0x00007f5995b26e63 in testing::UnitTest::Run (this=0x7f5995d461e0 <testing::UnitTest::GetInstance()::instance>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444
#13 0x00007f5996cc81de in RUN_ALL_TESTS () at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#14 0x00007f5996cc753d in main (argc=1, argv=0x7ffcd4583248) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:109
#15 0x00007f5993f17bf7 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#16 0x00005650d13c4e4a in _start ()
W20240502 21:00:29.801702 11023 meta_cache.cc:302] tablet e1dc93b317234f16bd5dd4b7dc2202cd: replica 168ea3788aeb40419f31ec54a55d9ba6 (127.9.215.195:33493) has failed: Network error: Client connection negotiation failed: client connection to 127.9.215.195:33493: connect: Connection refused (error 111)
************************* END STACKS ***************************
F20240502 21:00:29.894151 10082 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1714683629 (unix time) try "date -d @1714683629" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e80000275f) received by PID 10079 (TID 0x7f59876b8700) from PID 10079; stack trace: ***
@ 0x7f5995d97980 (unknown) at ??:0
@ 0x7f5993f34fb7 gsignal at ??:0
@ 0x7f5993f36921 abort at ??:0
@ 0x7f5994c39dcd google::LogMessage::Fail() at ??:0
@ 0x7f5994c3db93 google::LogMessage::SendToLog() at ??:0
@ 0x7f5994c397cc google::LogMessage::Flush() at ??:0
@ 0x7f5994c3af59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f5996cc71c4 _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at ??:0
@ 0x7f5996cc7888 _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at ??:0
@ 0x7f5996cc75dc _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at ??:0
@ 0x7f5996cc7e6e _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at ??:0
@ 0x7f5996cc7e13 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at ??:0
@ 0x7f5996cc7dce _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at ??:0
@ 0x7f599495a6df (unknown) at ??:0
@ 0x7f5995d8c6db start_thread at ??:0
@ 0x7f599401771f clone at ??:0