[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from RaftConsensusStressITest
[ RUN ] RaftConsensusStressITest.RemoveReplaceInCycle
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240503 02:00:13.833434 861 test_util.cc:268] Using random seed: 345414862
I20240503 02:00:13.839641 861 ts_itest-base.cc:112] Starting cluster with:
I20240503 02:00:13.839689 861 ts_itest-base.cc:113] --------------
I20240503 02:00:13.839708 861 ts_itest-base.cc:114] 6 tablet servers
I20240503 02:00:13.839726 861 ts_itest-base.cc:115] 3 replicas per TS
I20240503 02:00:13.839743 861 ts_itest-base.cc:116] --------------
2024-05-03T02:00:13Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-03T02:00:13Z Disabled control of system clock
I20240503 02:00:13.848323 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.215.126:35999
--webserver_interface=127.0.215.126
--webserver_port=0
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.215.126:35999
--minloglevel=2
--raft_prepare_replacement_before_eviction=true
--max_create_tablets_per_ts=40 with env {}
I20240503 02:00:14.002573 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 873
I20240503 02:00:14.002733 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240503 02:00:14.051800 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.65:0
--local_ip_for_outbound_sockets=127.0.215.65
--webserver_interface=127.0.215.65
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:14.236863 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 964
I20240503 02:00:14.236958 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240503 02:00:14.239056 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.66:0
--local_ip_for_outbound_sockets=127.0.215.66
--webserver_interface=127.0.215.66
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:14.403378 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 1095
I20240503 02:00:14.403467 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240503 02:00:14.405401 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.67:0
--local_ip_for_outbound_sockets=127.0.215.67
--webserver_interface=127.0.215.67
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:14.570533 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 1226
I20240503 02:00:14.570614 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240503 02:00:14.572266 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.68:0
--local_ip_for_outbound_sockets=127.0.215.68
--webserver_interface=127.0.215.68
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:14.726177 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 1357
I20240503 02:00:14.726271 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240503 02:00:14.728106 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.69:0
--local_ip_for_outbound_sockets=127.0.215.69
--webserver_interface=127.0.215.69
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:14.892387 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 1488
I20240503 02:00:14.892488 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240503 02:00:14.894297 861 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2OxuZi/build/debug/bin/kudu
/tmp/dist-test-task2OxuZi/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.215.70:0
--local_ip_for_outbound_sockets=127.0.215.70
--webserver_interface=127.0.215.70
--webserver_port=0
--tserver_master_addrs=127.0.215.126:35999
--builtin_ntp_servers=127.0.215.84:46643
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 02:00:15.058885 861 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2OxuZi/build/debug/bin/kudu as pid 1619
I20240503 02:00:15.058984 861 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240503 02:00:15.061328 861 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240503 02:00:15.180619 861 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240503 02:00:15.182015 861 ts_itest-base.cc:233] Waiting for 0 tablets on tserver b6f6530799ed4f1c897ee07b81707e17 to finish bootstrapping
I20240503 02:00:15.187053 861 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 762419d6a2314097bbbd24a41505af0a to finish bootstrapping
I20240503 02:00:15.191100 861 ts_itest-base.cc:233] Waiting for 1 tablets on tserver e5cf3a61ea3848a9a15513289b8bf476 to finish bootstrapping
I20240503 02:00:15.195307 861 ts_itest-base.cc:233] Waiting for 1 tablets on tserver e834766ed7b5410baf6fa83f5dfc43b9 to finish bootstrapping
I20240503 02:00:15.199345 861 ts_itest-base.cc:233] Waiting for 0 tablets on tserver b9327ad84d3f465a91f50184d524b05d to finish bootstrapping
I20240503 02:00:15.203976 861 ts_itest-base.cc:233] Waiting for 0 tablets on tserver cfb136abf320446887236da79a140f7d to finish bootstrapping
I20240503 02:00:15.209118 861 test_util.cc:268] Using random seed: 346790555
F20240503 02:00:24.073359 2302 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us, tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us) (tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.run_cpu_time_us, tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.run_cpu_time_us) (tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.run_wall_time_us, tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.run_wall_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.queue_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.queue_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_cpu_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_cpu_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_wall_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_wall_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us) (tablet-download-pool-a1cebbd32a0d4a5ea23ff6eb31388b18.queue_time_us, tablet-download-pool-a1cebbd32a0d4a5ea23ff6eb31388b18.queue_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us) (tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.queue_time_us, tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.queue_time_us) (tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.run_cpu_time_us, tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.run_cpu_time_us) (tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.run_wall_time_us, tablet-download-pool-d3d85299ee6d4b52a1bd5ce44e3b6b0d.run_wall_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.queue_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.queue_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_cpu_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_cpu_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_wall_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_wall_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f3d5f730dcd google::LogMessage::Fail() at ??:0
@ 0x7f3d5f734b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f3d5f7307cc google::LogMessage::Flush() at ??:0
@ 0x7f3d5f731f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f3d60b9a044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f3d60b7ef98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f3d60b7cefc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f3d6372f77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f3d63738a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f3d637b976f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f3d637cd4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f3d637b8226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f3d637c7c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55b11d3a075c std::function<>::operator()() at ??:0
@ 0x7f3d60b81e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f3d60b826ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f3d60b8403b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55b11d3a075c std::function<>::operator()() at ??:0
@ 0x7f3d60b71807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f3d613af6db start_thread at ??:0
@ 0x7f3d5e8e771f clone at ??:0
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 0 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | HEALTHY | <none> | 6 | 0
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 14 | 0
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | HEALTHY | <none> | 10 | 0
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
local_ip_for_outbound_sockets | 127.0.215.65 | experimental | 127.0.215.65:46459
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.68 | experimental | 127.0.215.68:34223
local_ip_for_outbound_sockets | 127.0.215.70 | experimental | 127.0.215.70:37631
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.215.65:46459
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.0.215.68:34223
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.215.70:37631
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+-----------------------------------------------------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
time_source | builtin | 127.0.215.66:43497, 127.0.215.65:46459, 127.0.215.70:37631, and 2 other server(s)
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.67:41355, tserver@127.0.215.66:43497, and 3 other server(s)
Tablet Summary
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A B C | 2 | -1 | Yes
B | [config not available] | | |
C | A* B C | 1 | -1 | Yes
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 58 | No
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 72 | No
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 65 | No
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 2 | 62 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 2 | 59 | No
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 66 | Yes
B | A B* C | 1 | 66 | Yes
C | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | -1 | Yes
B | [config not available] | | |
C | A B C* | 2 | -1 | Yes
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 64 | Yes
B | A B* C | 1 | 64 | Yes
C | [config not available] | | |
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 74 | Yes
B | A B* C | 1 | 74 | Yes
C | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 2 | 60 | No
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 68 | Yes
B | A B* C | 1 | 68 | Yes
C | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | -1 | Yes
B | [config not available] | | |
C | A B C* | 2 | -1 | Yes
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 65 | No
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 62 | No
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 67 | Yes
B | A B* C | 1 | 67 | Yes
C | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 61 | Yes
B | A* B C | 1 | 61 | Yes
C | [config not available] | | |
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* D~ | 1 | 63 | No
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
D = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | -1 | Yes
B | [config not available] | | |
C | A* B C D~ | 2 | 56 | No
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = b9327ad84d3f465a91f50184d524b05d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 57 | Yes
B | A B* C | 2 | 57 | Yes
C | [config not available] | | |
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): RUNNING
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): RUNNING [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNDER_REPLICATED | 40 | 13 | 0 | 27 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 13
First Quartile | 20
Median | 21
Third Quartile | 21
Maximum | 27
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
-------+----------------------------------+--------------------+---------------
Big | b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | 27
Small | 762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | 13
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 123
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
F20240503 02:00:28.045459 2810 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.queue_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.queue_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_cpu_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_cpu_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_wall_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_wall_time_us) (tablet-download-pool-2393873a64ac438eb519c75ef956392e.queue_time_us, tablet-download-pool-2393873a64ac438eb519c75ef956392e.queue_time_us) (tablet-download-pool-2393873a64ac438eb519c75ef956392e.run_cpu_time_us, tablet-download-pool-2393873a64ac438eb519c75ef956392e.run_cpu_time_us) (tablet-download-pool-2393873a64ac438eb519c75ef956392e.run_wall_time_us, tablet-download-pool-2393873a64ac438eb519c75ef956392e.run_wall_time_us) (tablet-download-pool-2ecd29082b0349e09024f19572fd3658.queue_time_us, tablet-download-pool-2ecd29082b0349e09024f19572fd3658.queue_time_us) (tablet-download-pool-2ecd29082b0349e09024f19572fd3658.run_cpu_time_us, tablet-download-pool-2ecd29082b0349e09024f19572fd3658.run_cpu_time_us) (tablet-download-pool-2ecd29082b0349e09024f19572fd3658.run_wall_time_us, tablet-download-pool-2ecd29082b0349e09024f19572fd3658.run_wall_time_us) (tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.queue_time_us, tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.queue_time_us) (tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.run_cpu_time_us, tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.run_cpu_time_us) (tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.run_wall_time_us, tablet-download-pool-32068ed1ba7944c291d3c02519cef12e.run_wall_time_us) (tablet-download-pool-49355da3d17240efb3dbce8678c11da3.queue_time_us, tablet-download-pool-49355da3d17240efb3dbce8678c11da3.queue_time_us) (tablet-download-pool-49355da3d17240efb3dbce8678c11da3.run_cpu_time_us, tablet-download-pool-49355da3d17240efb3dbce8678c11da3.run_cpu_time_us) (tablet-download-pool-49355da3d17240efb3dbce8678c11da3.run_wall_time_us, tablet-download-pool-49355da3d17240efb3dbce8678c11da3.run_wall_time_us) (tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.queue_time_us, tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.queue_time_us) (tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.run_cpu_time_us, tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.run_cpu_time_us) (tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.run_wall_time_us, tablet-download-pool-63ff5b719ceb48e8afa05fa537615a57.run_wall_time_us) (tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.queue_time_us, tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.queue_time_us) (tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.run_cpu_time_us, tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.run_cpu_time_us) (tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.run_wall_time_us, tablet-download-pool-63ffc9287fbf44e2abcb8fc2a57bf15e.run_wall_time_us) (tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.queue_time_us, tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.queue_time_us) (tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.run_cpu_time_us, tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.run_cpu_time_us) (tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.run_wall_time_us, tablet-download-pool-94c57e6abc584aeab4597fd695fdb158.run_wall_time_us) (tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.queue_time_us, tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.queue_time_us) (tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.run_cpu_time_us, tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.run_cpu_time_us) (tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.run_wall_time_us, tablet-download-pool-9d51804ae96d4ba48b4b0fb99bd4c3a9.run_wall_time_us) (tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.queue_time_us, tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.queue_time_us) (tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.run_cpu_time_us, tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.run_cpu_time_us) (tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.run_wall_time_us, tablet-download-pool-a001804c08de4aa9a989fd399bcbab85.run_wall_time_us) (tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.queue_time_us, tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.queue_time_us) (tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.run_cpu_time_us, tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.run_cpu_time_us) (tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.run_wall_time_us, tablet-download-pool-c4d0e0e44f0c45a7ad25062477500b85.run_wall_time_us) (tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.queue_time_us, tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.queue_time_us) (tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.run_cpu_time_us, tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.run_cpu_time_us) (tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.run_wall_time_us, tablet-download-pool-d0d072cddb44437db2335f82da6cfe76.run_wall_time_us) (tablet-download-pool-daf20382e45943ca955bf509d5724f1d.queue_time_us, tablet-download-pool-daf20382e45943ca955bf509d5724f1d.queue_time_us) (tablet-download-pool-daf20382e45943ca955bf509d5724f1d.run_cpu_time_us, tablet-download-pool-daf20382e45943ca955bf509d5724f1d.run_cpu_time_us) (tablet-download-pool-daf20382e45943ca955bf509d5724f1d.run_wall_time_us, tablet-download-pool-daf20382e45943ca955bf509d5724f1d.run_wall_time_us) (tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.queue_time_us, tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.queue_time_us) (tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.run_cpu_time_us, tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.run_cpu_time_us) (tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.run_wall_time_us, tablet-download-pool-dd0b8470162c4f19a4d0080f7b4dfc2a.run_wall_time_us) (tablet-download-pool-e669c5827e714000a4521d9e1c1ba55d.queue_time_us, tablet-download-pool-e669c5827e714000a4521d9e1c1ba55d.queue_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fdefc977dcd google::LogMessage::Fail() at ??:0
@ 0x7fdefc97bb93 google::LogMessage::SendToLog() at ??:0
@ 0x7fdefc9777cc google::LogMessage::Flush() at ??:0
@ 0x7fdefc978f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fdefdde1044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fdefddc5f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fdefddc3efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fdf0097677a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fdf0097fa45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fdf00a0076f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fdf00a144f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fdf009ff226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fdf00a0ec0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5640aade075c std::function<>::operator()() at ??:0
@ 0x7fdefddc8e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fdefddc96ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fdefddcb03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5640aade075c std::function<>::operator()() at ??:0
@ 0x7fdefddb8807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fdefe5f66db start_thread at ??:0
@ 0x7fdefbb2e71f clone at ??:0
W20240503 02:00:28.151566 1785 meta_cache.cc:302] tablet 32068ed1ba7944c291d3c02519cef12e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:28.294417 1785 connection.cc:535] client connection to 127.0.215.65:46459 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240503 02:00:29.180028 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:30.219208 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:31.274945 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:32.275779 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:33.415733 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:34.482978 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:35.527546 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:36.655071 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:37.723309 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:38.992668 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:40.048677 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:41.310330 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:42.459933 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:43.490108 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:44.553436 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:45.654898 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:46.790232 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:47.960759 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:00:48.967725 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:50.198168 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:51.472817 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:52.779837 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:54.124250 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:55.502943 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:56.919862 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:58.372231 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:00:59.861639 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:00.878895 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:02.172207 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:03.221990 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:04.567265 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:05.653900 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:07.036331 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:08.157084 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:09.584682 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:10.744508 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:12.212726 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:13.403789 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:14.915756 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:16.145622 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:17.713205 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:18.980247 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:20.588666 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:21.890239 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:23.538396 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:24.538614 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
F20240503 02:01:26.105639 6648 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us, tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.queue_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.queue_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_cpu_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_cpu_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_wall_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_wall_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.queue_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.queue_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_cpu_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_cpu_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_wall_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_wall_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.queue_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.queue_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_cpu_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_cpu_time_us) (tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_wall_time_us, tablet-download-pool-3ca33aff4f8943a1b1822500cad0bc74.run_wall_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.queue_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.queue_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_cpu_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_cpu_time_us) (tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_wall_time_us, tablet-download-pool-d686ef1a0f784fee95998b824ca4e4bd.run_wall_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f4590525dcd google::LogMessage::Fail() at ??:0
@ 0x7f4590529b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f45905257cc google::LogMessage::Flush() at ??:0
@ 0x7f4590526f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f459198f044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f4591973f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f4591971efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f459452477a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f459452da45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f45945ae76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f45945c24f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f45945ad226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f45945bcc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x558f7051975c std::function<>::operator()() at ??:0
@ 0x7f4591976e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f45919776ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f459197903b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x558f7051975c std::function<>::operator()() at ??:0
@ 0x7f4591966807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f45921a46db start_thread at ??:0
@ 0x7f458f6dc71f clone at ??:0
W20240503 02:01:26.579422 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:27.600701 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:28.283289 1785 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet d874897a3e1547cbae6f34bd121856ac after 340 attempt(s): Failed to write to server: (no server available): Write(tablet: d874897a3e1547cbae6f34bd121856ac, num_ops: 2, num_attempts: 340) passed its deadline: Illegal state: replica cfb136abf320446887236da79a140f7d is not leader of this config: current role FOLLOWER
W20240503 02:01:28.647549 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:29.718423 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:30.845242 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:31.848284 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:32.930544 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:34.051054 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:35.061533 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:36.138815 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:37.304383 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:38.554579 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:39.582976 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:40.814636 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:41.941429 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:42.953207 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:44.002034 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:45.087909 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:46.201799 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:47.354403 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:48.546139 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:01:49.574944 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:50.829481 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:52.117501 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:53.445295 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:54.816247 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:56.222100 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:57.655750 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:01:59.132369 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:00.639767 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:01.661430 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:02.965566 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:04.026327 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:05.376781 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:06.471431 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:07.866592 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:08.998064 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:10.433669 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:11.604353 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:13.092442 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:14.305424 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:15.842934 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:17.083289 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:18.657670 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:19.935994 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:21.560297 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:22.876771 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:24.539928 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:25.552853 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:26.572103 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
F20240503 02:02:27.131415 8300 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us, tablet-download-pool-01b1f2ff446142aa9a42dcb2ec22c9a2.queue_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.queue_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_cpu_time_us) (tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us, tablet-download-pool-03c261a02819489b836b54f8264d63c1.run_wall_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.queue_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.queue_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_cpu_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_cpu_time_us) (tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_wall_time_us, tablet-download-pool-0946ea6710e64c4ba2b6292cca0d021d.run_wall_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.queue_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_cpu_time_us) (tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us, tablet-download-pool-0a60ff4ed2614ca380d65c6c3916b036.run_wall_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.queue_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_cpu_time_us) (tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us, tablet-download-pool-0da3dfe44b9f4c2eadc3b1b5d07ded2c.run_wall_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.queue_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.queue_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_cpu_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_cpu_time_us) (tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_wall_time_us, tablet-download-pool-16010499f8f44f8e926d8a57da62f7d5.run_wall_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.queue_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_cpu_time_us) (tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us, tablet-download-pool-2076984e76f14b488c923803974c2f12.run_wall_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.queue_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.queue_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_cpu_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_cpu_time_us) (tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_wall_time_us, tablet-download-pool-2cb7a8644d4c45fabebb084a5b0c3420.run_wall_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.queue_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_cpu_time_us) (tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us, tablet-download-pool-4599093b65dd489fb364b4d89a94142b.run_wall_time_us) (tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.queue_time_us, tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.queue_time_us) (tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.run_cpu_time_us, tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.run_cpu_time_us) (tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.run_wall_time_us, tablet-download-pool-932ca9342c83412398c9b6ce755e7daf.run_wall_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.queue_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_cpu_time_us) (tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us, tablet-download-pool-9db610e3a63c445eb731bb482995879c.run_wall_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.queue_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_cpu_time_us) (tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us, tablet-download-pool-a2815297d51a4f41b15756427a372423.run_wall_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.queue_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_cpu_time_us) (tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us, tablet-download-pool-d93b96a6653d4c2f887c73e3b5ee452c.run_wall_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.queue_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_cpu_time_us) (tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us, tablet-download-pool-e3e3cc25f73446cc8492aa56e3d2b11a.run_wall_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.queue_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_cpu_time_us) (tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us, tablet-download-pool-fea563f16a7c4bbb95b5ceac4a00cdea.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f53f3a2ddcd google::LogMessage::Fail() at ??:0
@ 0x7f53f3a31b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f53f3a2d7cc google::LogMessage::Flush() at ??:0
@ 0x7f53f3a2ef59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f53f4e97044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f53f4e7bf98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f53f4e79efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f53f7a2c77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f53f7a35a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f53f7ab676f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f53f7aca4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f53f7ab5226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f53f7ac4c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x557fb8ea675c std::function<>::operator()() at ??:0
@ 0x7f53f4e7ee06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f53f4e7f6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f53f4e8103b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x557fb8ea675c std::function<>::operator()() at ??:0
@ 0x7f53f4e6e807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f53f56ac6db start_thread at ??:0
@ 0x7f53f2be471f clone at ??:0
W20240503 02:02:27.338086 1783 connection.cc:535] client connection to 127.0.215.70:37631 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240503 02:02:27.572516 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:28.286581 1784 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 03c261a02819489b836b54f8264d63c1 after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.996s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.67:41355, tserver@127.0.215.66:43497
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 173 | Yes
C | A B C* | 1 | 173 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 175 | Yes
C | A B C* D~ | 2 | 175 | Yes
D | [config not available] | | |
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 175 | Yes
C | A B* C | 2 | 175 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 177 | Yes
C | A B C* | 2 | 177 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 180 | Yes
C | A B C* | 3 | 180 | Yes
D | [config not available] | | |
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 36
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 130
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:02:28.608364 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:29.613538 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:30.626042 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:31.636322 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:32.646514 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:33.657106 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:34.659763 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:35.674571 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:36.730042 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:37.780717 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:38.811280 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:39.836792 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:02:40.847358 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:41.868294 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:42.869684 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:43.887255 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:44.892618 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:45.913455 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:46.937036 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:47.974407 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:48.981418 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:50.000208 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:51.047744 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:52.059844 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:02:53.139849 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:54.164574 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:55.179636 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:56.183039 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:57.299314 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:02:58.339849 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:02:59.367892 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:00.503590 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:01.522776 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:02.559983 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:03.613394 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:04.680573 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:05.763612 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:06.863528 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:07.982879 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:09.111913 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:10.258424 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:11.273038 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:12.304875 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:13.341038 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:14.402442 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:15.457827 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:16.546176 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:17.559194 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:18.572052 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:19.696781 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:20.696990 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:21.712631 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:22.759289 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:23.940023 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:24.940908 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:25.949685 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:26.969234 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:27.997146 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:28.609601 1784 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 4599093b65dd489fb364b4d89a94142b after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.997s (SENT)
W20240503 02:03:28.997565 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:30.020969 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:31.068293 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:32.091140 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:33.135437 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:34.147581 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:35.183130 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:36.231691 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:37.242934 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:38.314806 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:39.332494 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:40.337455 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:41.422410 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:42.434218 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:43.441148 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:44.464725 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:45.512244 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:46.621950 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:47.631528 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:48.635633 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:49.646000 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:50.654734 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:51.672571 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:52.729033 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:53.837234 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:03:54.967883 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:55.971220 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:57.065757 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:03:58.083652 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:03:59.086606 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:00.236954 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:01.252808 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:02.281064 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:03.317699 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:04.376659 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:05.450212 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:06.540639 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:07.647122 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:08.771697 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:09.909686 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:11.063774 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:12.238777 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:13.429239 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:14.438357 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:15.546702 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:16.556211 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:17.707242 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:18.740743 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:19.917048 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:20.924865 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:21.947494 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:22.949446 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:23.959146 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:24.961236 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:25.970503 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:26.970767 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:27.974318 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:28.842978 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.998s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.66:43497, tserver@127.0.215.67:41355
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:04:29.290272 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:30.297075 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:31.304291 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:32.322767 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:33.325316 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:34.341202 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:35.347391 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:36.356550 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:37.361630 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:38.381014 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:39.385790 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:40.394137 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:41.398057 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:42.416514 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:43.421034 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:44.441962 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:45.453294 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:46.479058 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:47.485091 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:48.517266 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:49.580170 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:50.601092 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:04:51.643363 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:52.643771 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:53.665496 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:54.700536 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:55.761061 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:56.767736 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:57.772156 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:04:58.832499 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:04:59.895428 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:00.939252 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:01.949394 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:02.979089 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:03.987905 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:05.048861 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:06.129876 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:07.222404 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:08.230329 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:09.244156 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:10.319082 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:11.358520 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:12.378405 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:13.412179 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:14.430462 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:15.510396 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:16.555858 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:17.670029 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:18.731199 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:19.875820 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:05:20.958513 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:21.968403 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:23.106007 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:24.221166 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:25.250487 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:26.258322 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:27.274716 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:28.299170 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:29.290009 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 03c261a02819489b836b54f8264d63c1 after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.997s (SENT)
W20240503 02:05:29.529271 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:30.531687 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:31.552860 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:32.554894 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:33.555014 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:34.589051 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:35.595736 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:36.605562 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:37.612289 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:38.632864 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:05:39.640861 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:40.661137 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:41.678857 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:05:42.684239 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:43.744354 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:44.772652 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:45.834431 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:46.858819 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:47.875486 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:48.889212 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:49.893270 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:50.907483 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:51.923807 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:52.973165 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:53.979435 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:54.979568 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:55.983700 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:05:57.025035 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:05:58.034198 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:05:59.050858 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:00.065129 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:01.154429 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:02.169390 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:03.201018 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:04.250247 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:05.314193 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:06.392048 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:07.478560 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:08.482686 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:09.500730 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:10.574903 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:11.578461 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:12.606926 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:13.627621 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:14.684638 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:15.708823 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:16.811165 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:17.858757 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:18.989426 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:19.999835 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:21.023519 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:06:22.031878 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:23.164690 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:24.184146 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:25.198941 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:26.203446 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:27.218014 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:28.229019 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:29.230413 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:29.530128 1784 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 03c261a02819489b836b54f8264d63c1 after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 60.000s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.66:43497, tserver@127.0.215.67:41355
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:06:30.233186 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:31.239861 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:32.253105 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:33.269524 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:06:34.281107 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:35.281713 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:36.296797 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:37.301826 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:38.359688 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:39.374032 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:40.424258 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:41.460218 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:42.470943 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:43.514727 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:44.521543 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:45.562927 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:46.638337 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:47.659005 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:48.709411 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:49.712548 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:50.716555 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:51.725754 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:52.784837 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:06:53.863004 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:06:54.970189 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:55.987408 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:06:57.026252 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:58.031265 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:06:59.036239 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:00.138703 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:01.145074 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:02.152491 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:03.166319 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:04.206147 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:05.260840 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:06.330586 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:07.417321 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:08.516769 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:09.630255 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:10.769367 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:11.915895 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:12.926308 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:13.968220 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:15.002919 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:16.072161 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:17.081630 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:18.215390 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:19.229641 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:20.242389 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:21.373744 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:22.385320 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:23.386884 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:24.409621 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:25.437381 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:26.440701 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:27.455394 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:28.476763 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:29.502851 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:29.971745 1784 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 4599093b65dd489fb364b4d89a94142b after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.998s (SENT)
W20240503 02:07:30.502948 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:31.505620 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:32.516141 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:33.535516 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:34.546414 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:35.567198 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:36.571586 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:37.595671 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:38.609140 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:39.636101 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:40.651249 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:41.651516 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:42.671169 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:43.674767 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:44.680796 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:45.713667 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:46.718817 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:47.747680 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:48.747812 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:49.751586 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:50.752027 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:51.766043 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:07:52.785336 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:53.828953 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:54.829524 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:55.857764 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:56.868752 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:07:57.905308 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:07:58.919703 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:07:59.955364 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:00.969535 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:01.970615 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:02.985791 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:04.015501 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:05.029086 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:08:06.095571 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:07.103329 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:08.126552 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:09.133504 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:10.145835 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:11.225576 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:12.246455 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:13.258039 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:14.301926 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:08:15.342535 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:16.407936 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:17.469738 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:18.558516 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:19.644910 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:20.655647 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:21.719061 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:08:22.719636 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:23.819715 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:24.829434 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:25.842171 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:26.847565 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:27.850247 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:28.852941 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:29.877440 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:30.202164 1784 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 4599093b65dd489fb364b4d89a94142b after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.991s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.66:43497, tserver@127.0.215.67:41355
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:08:30.879767 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:31.881449 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:32.898113 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:33.926090 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:34.929452 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:35.948859 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:36.962105 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:37.964360 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:38.969411 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:40.022590 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:41.035594 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:42.037413 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:43.108312 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:44.111937 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:45.131597 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:46.134831 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:47.179693 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:48.183388 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:49.241151 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:50.245847 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:51.246198 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:52.253893 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:53.306689 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:54.381712 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:08:55.482432 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:56.607560 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:57.611656 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:08:58.688928 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:08:59.723100 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:00.723109 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:01.842132 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:02.849611 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:03.870781 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:04.905836 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:05.960266 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:07.035935 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:08.124114 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:09.226752 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:10.342397 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:11.479792 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:12.625908 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:13.787748 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:14.793159 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:15.870564 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:16.891346 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:18.000792 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:19.052491 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:20.175948 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:21.251662 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:22.402170 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:23.502379 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:24.677358 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:25.688486 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:26.688699 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:27.695751 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:28.699556 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:29.709558 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:30.674399 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet fea563f16a7c4bbb95b5ceac4a00cdea after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.998s (SENT)
W20240503 02:09:30.906562 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:31.906817 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:32.916771 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:33.923039 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:34.924665 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:35.947789 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:36.958009 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:37.959169 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:38.960453 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:39.969400 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:40.982666 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:42.005894 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:43.007874 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:44.021842 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:45.091722 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:46.125993 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:47.190109 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:48.202661 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:49.234611 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:50.249213 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:51.269917 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:52.295044 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:53.312167 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:54.347637 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:55.445469 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:56.451300 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:09:57.486073 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:09:58.486447 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:09:59.604758 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:00.626747 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:01.657359 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:02.671506 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:03.688326 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:04.698274 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:05.709689 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:06.712509 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:07.787346 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:08.880223 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:09.992069 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:11.118175 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:12.264191 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:13.267833 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:14.296237 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:15.296742 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:16.376757 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:17.379055 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:18.507053 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:19.507629 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:20.552682 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:21.636687 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:22.666754 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:23.689764 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:24.733419 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:25.858002 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:26.858407 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:27.866381 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:28.888424 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:29.917824 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:30.905500 1784 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 03c261a02819489b836b54f8264d63c1 after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.996s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.67:41355, tserver@127.0.215.66:43497
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:10:31.374910 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:32.383349 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:33.388185 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:34.389369 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:35.394097 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:36.419553 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:37.437038 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:38.438315 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:39.455231 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:40.461558 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:10:41.478626 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:42.494576 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:43.508025 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:44.510973 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:45.571223 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:46.596545 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:47.665117 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:48.677937 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:10:49.718084 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:50.743814 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:51.746191 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:52.771698 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:53.772876 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:54.819258 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:10:55.912993 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:56.927031 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:10:57.951540 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:10:58.968874 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:00.073140 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:01.128551 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:02.145392 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:03.146122 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:04.153859 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:05.159559 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:06.206522 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:07.272425 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:08.276768 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:09.332890 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:10.445525 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:11.574251 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:12.575697 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:13.588166 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:14.589998 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:15.648674 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:16.658457 1785 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:17.756426 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:18.798270 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:19.909607 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:20.970957 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:22.113655 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:23.203243 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:24.213204 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:25.351459 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:26.356655 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:27.373358 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:28.379139 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:29.382344 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:30.391024 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:31.374670 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.996s (SENT)
W20240503 02:11:31.625005 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:32.628630 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:33.643218 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:34.651788 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:35.655154 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:36.682799 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:37.733069 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:38.762288 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:39.784031 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:40.799901 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:41.811074 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:42.838901 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:43.871114 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:44.873939 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:45.994151 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:47.026342 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:48.098655 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:49.204264 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:50.207556 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:51.328192 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:52.333431 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:53.362638 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:54.413041 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:55.493031 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:56.600760 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:11:57.727192 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:11:58.736932 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:11:59.822991 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:00.845009 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:01.853570 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:02.857754 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:03.866386 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:04.866407 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:05.879901 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:06.913379 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:07.933415 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:09.016438 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:10.123739 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:11.241132 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:12.371202 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:13.521057 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:14.522573 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:15.577232 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:16.580176 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:17.675660 1782 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:18.712317 1783 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:19.827764 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:20.829216 1785 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:21.838292 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:22.994369 1783 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:24.020895 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:25.054312 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:26.061967 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:27.067572 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:28.070977 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:29.087611 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:30.091893 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:31.098259 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:31.623790 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.995s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.66:43497, tserver@127.0.215.67:41355
Tablet Summary
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:12:32.099026 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:33.120430 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:34.120821 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:35.122859 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:36.132447 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:37.147625 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:38.154913 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:39.175810 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:40.200146 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:41.205550 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:42.227689 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:43.251614 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:44.264461 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:45.276216 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:46.288086 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:47.300200 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:48.300655 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:12:49.306028 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:50.338815 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:51.386152 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:52.389452 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:53.403105 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:54.449061 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:12:55.450462 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:56.465430 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:12:57.509325 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:58.554293 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:12:59.573557 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:00.599635 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:01.630101 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:13:02.707887 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:03.727555 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:04.740932 1785 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:05.751950 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:06.794595 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:07.860846 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:08.938237 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:10.034497 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:11.042402 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111)
W20240503 02:13:12.071425 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:13.125484 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:14.170680 1785 meta_cache.cc:302] tablet d874897a3e1547cbae6f34bd121856ac: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:15.170857 1782 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:16.228937 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:17.230654 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:18.239113 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:19.249513 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:20.274857 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:21.329603 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:22.370745 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:23.433660 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:24.455698 1782 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:25.503346 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:26.579244 1785 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:27.683920 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:28.689092 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:29.707441 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:30.733119 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:31.765615 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:32.077939 1784 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 4599093b65dd489fb364b4d89a94142b after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.992s (SENT)
W20240503 02:13:32.766441 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:33.776238 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:34.776893 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:35.780301 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:36.807807 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:37.818557 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:38.819659 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:39.850230 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:40.863219 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:41.880510 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:42.896179 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:43.915674 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:44.922102 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:45.946597 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:46.956061 1782 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:47.986610 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:49.015350 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:50.031450 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:51.090570 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:52.113734 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:53.118686 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:13:54.145159 1785 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:55.156903 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:13:56.165091 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:57.165105 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:58.171289 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:13:59.175048 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:00.178282 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:01.215122 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:02.270367 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:03.275039 1783 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:04.276007 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:05.279520 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:06.308813 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:07.358800 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:08.428194 1785 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:09.512876 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:10.609763 1782 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:11.613003 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:12.626717 1783 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:13.715754 1783 meta_cache.cc:302] tablet e669c5827e714000a4521d9e1c1ba55d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:14.758029 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:15.759140 1782 meta_cache.cc:302] tablet eb3c885127884ccd86b4be3677f1f4b8: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:16.820659 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:17.840009 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:18.842831 1782 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:19.859022 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:20.903276 1782 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:21.940212 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:22.982550 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:24.070026 1785 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:25.085564 1785 meta_cache.cc:302] tablet dd0b8470162c4f19a4d0080f7b4dfc2a: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:26.120616 1783 meta_cache.cc:302] tablet 31f2fe51d3894b7da6e322baac517174: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:27.193450 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:28.327998 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:29.338102 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:30.354391 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:31.385182 1783 meta_cache.cc:302] tablet 63ffc9287fbf44e2abcb8fc2a57bf15e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:32.347537 1784 batcher.cc:441] Timed out: Failed to write batch of 4 ops to tablet fea563f16a7c4bbb95b5ceac4a00cdea after 1 attempt(s): Failed to write to server: e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): Write RPC to 127.0.215.67:41355 timed out after 59.999s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+---------------------+---------
0df44c341f7d48159420bf2a8c5b0b04 | 127.0.215.126:35999 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+--------------------+-------------------------
builtin_ntp_servers | 127.0.215.84:46643 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+--------------------+-------------+----------+----------------+-----------------
762419d6a2314097bbbd24a41505af0a | 127.0.215.66:43497 | HEALTHY | <none> | 10 | 0
e5cf3a61ea3848a9a15513289b8bf476 | 127.0.215.67:41355 | HEALTHY | <none> | 20 | 0
b6f6530799ed4f1c897ee07b81707e17 | 127.0.215.65:46459 | UNAVAILABLE | <none> | n/a | n/a
b9327ad84d3f465a91f50184d524b05d | 127.0.215.69:42309 | UNAVAILABLE | <none> | n/a | n/a
cfb136abf320446887236da79a140f7d | 127.0.215.70:37631 | UNAVAILABLE | <none> | n/a | n/a
e834766ed7b5410baf6fa83f5dfc43b9 | 127.0.215.68:34223 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.215.65:46459: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.65:46459: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.69:42309: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.70:37631: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.215.68:34223: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+----------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.215.66:43497, 127.0.215.67:41355
local_ip_for_outbound_sockets | 127.0.215.66 | experimental | 127.0.215.66:43497
local_ip_for_outbound_sockets | 127.0.215.67 | experimental | 127.0.215.67:41355
never_fsync | true | unsafe,advanced | 127.0.215.66:43497, 127.0.215.67:41355
openssl_security_level_override | 0 | unsafe,hidden | 127.0.215.66:43497, 127.0.215.67:41355
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.215.66:43497, 127.0.215.67:41355
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_format | pb | hidden | 127.0.215.66:43497, 127.0.215.67:41355
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.215.66:43497
server_dump_info_path | /tmp/dist-test-task2OxuZi/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714701613823308-861-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.215.67:41355
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.215.66:43497, 127.0.215.67:41355
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+--------------------+----------------------------------------
builtin_ntp_servers | 127.0.215.84:46643 | 127.0.215.66:43497, 127.0.215.67:41355
time_source | builtin | 127.0.215.66:43497, 127.0.215.67:41355
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.215.126:35999, tserver@127.0.215.66:43497, tserver@127.0.215.67:41355
Tablet Summary
Tablet 16010499f8f44f8e926d8a57da62f7d5 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 12 | Yes
B | [config not available] | | |
C | A* B C | 1 | 12 | Yes
Tablet d874897a3e1547cbae6f34bd121856ac of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = b9327ad84d3f465a91f50184d524b05d
C = b6f6530799ed4f1c897ee07b81707e17
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 932ca9342c83412398c9b6ce755e7daf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 4 | 73 | Yes
B | A* B C | 4 | 73 | Yes
C | [config not available] | | |
Tablet 2cb7a8644d4c45fabebb084a5b0c3420 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 120 | Yes
B | A* B C | 3 | 120 | Yes
C | [config not available] | | |
Tablet 49355da3d17240efb3dbce8678c11da3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 71 | Yes
C | A B* C | 2 | 71 | Yes
Tablet 01b1f2ff446142aa9a42dcb2ec22c9a2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 230 | Yes
C | A B C* | 1 | 230 | Yes
Tablet daf20382e45943ca955bf509d5724f1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d3d85299ee6d4b52a1bd5ce44e3b6b0d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 238 | Yes
C | A B C* | 1 | 238 | Yes
Tablet 94c57e6abc584aeab4597fd695fdb158 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 85 | Yes
C | A B* C | 2 | 85 | Yes
Tablet d686ef1a0f784fee95998b824ca4e4bd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
D = e834766ed7b5410baf6fa83f5dfc43b9
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D | 3 | 122 | No
C | A B C* D | 3 | 122 | No
D | [config not available] | | |
Tablet 2ecd29082b0349e09024f19572fd3658 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 78 | Yes
C | A B* C | 2 | 78 | Yes
Tablet 9db610e3a63c445eb731bb482995879c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 3 | 121 | Yes
B | A B* C | 3 | 121 | Yes
C | [config not available] | | |
Tablet 63ff5b719ceb48e8afa05fa537615a57 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 73 | Yes
C | A B* C | 4 | 73 | Yes
Tablet d74c5e7bcc044ababe89975572f47183 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet dd0b8470162c4f19a4d0080f7b4dfc2a of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3ca33aff4f8943a1b1822500cad0bc74 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 231 | Yes
C | A B* C | 2 | 231 | Yes
Tablet 32068ed1ba7944c291d3c02519cef12e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 71 | Yes
C | A B C* | 4 | 71 | Yes
Tablet 0da3dfe44b9f4c2eadc3b1b5d07ded2c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 126 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet cb3cca27ce284a7882131479b745e61e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 2076984e76f14b488c923803974c2f12 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 128 | Yes
B | [config not available] | | |
C | A* B C | 1 | 128 | Yes
Tablet eb3c885127884ccd86b4be3677f1f4b8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 03c261a02819489b836b54f8264d63c1 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 134 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 2393873a64ac438eb519c75ef956392e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 72 | Yes
C | A B C* | 4 | 72 | Yes
Tablet 4599093b65dd489fb364b4d89a94142b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 128 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet e669c5827e714000a4521d9e1c1ba55d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e3e3cc25f73446cc8492aa56e3d2b11a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 126 | Yes
B | A B* C | 1 | 126 | Yes
C | [config not available] | | |
Tablet ef979e6ffc7242bea2ca48b384720b40 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet d93b96a6653d4c2f887c73e3b5ee452c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 122 | Yes
B | A B* C | 2 | 122 | Yes
C | [config not available] | | |
Tablet a001804c08de4aa9a989fd399bcbab85 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 28d5c8cce38e48258731fbbfb0adf961 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 233 | Yes
C | A B C* | 2 | 233 | Yes
Tablet c4d0e0e44f0c45a7ad25062477500b85 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 77 | Yes
C | A B* C | 2 | 77 | Yes
Tablet 0946ea6710e64c4ba2b6292cca0d021d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
All reported replicas are:
A = 762419d6a2314097bbbd24a41505af0a
B = e5cf3a61ea3848a9a15513289b8bf476
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 127 | Yes
B | A* B C | 3 | 127 | Yes
C | [config not available] | | |
Tablet 9d51804ae96d4ba48b4b0fb99bd4c3a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING [LEADER]
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 74 | Yes
C | A B* C | 3 | 74 | Yes
Tablet fea563f16a7c4bbb95b5ceac4a00cdea of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [NONVOTER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = e5cf3a61ea3848a9a15513289b8bf476
C = e834766ed7b5410baf6fa83f5dfc43b9
D = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D | 1 | 127 | No
C | [config not available] | | |
D | [config not available] | | |
Tablet 31f2fe51d3894b7da6e322baac517174 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 0a60ff4ed2614ca380d65c6c3916b036 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | [config not available] | | |
C | A* B C | 2 | 127 | Yes
Tablet 63ffc9287fbf44e2abcb8fc2a57bf15e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a1cebbd32a0d4a5ea23ff6eb31388b18 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
b6f6530799ed4f1c897ee07b81707e17 (127.0.215.65:46459): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = b6f6530799ed4f1c897ee07b81707e17
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 237 | Yes
C | A B C* | 3 | 237 | Yes
Tablet d0d072cddb44437db2335f82da6cfe76 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
All reported replicas are:
A = cfb136abf320446887236da79a140f7d
B = 762419d6a2314097bbbd24a41505af0a
C = e5cf3a61ea3848a9a15513289b8bf476
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 69 | Yes
C | A B C* | 4 | 69 | Yes
Tablet a2815297d51a4f41b15756427a372423 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
e5cf3a61ea3848a9a15513289b8bf476 (127.0.215.67:41355): RUNNING [LEADER]
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable
762419d6a2314097bbbd24a41505af0a (127.0.215.66:43497): RUNNING
All reported replicas are:
A = e5cf3a61ea3848a9a15513289b8bf476
B = cfb136abf320446887236da79a140f7d
C = 762419d6a2314097bbbd24a41505af0a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 123 | Yes
B | [config not available] | | |
C | A* B C | 2 | 123 | Yes
Tablet 967cdb765da9410f89920ace401c5f51 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223): TS unavailable
b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309): TS unavailable
cfb136abf320446887236da79a140f7d (127.0.215.70:37631): TS unavailable [LEADER]
All reported replicas are:
A = e834766ed7b5410baf6fa83f5dfc43b9
B = b9327ad84d3f465a91f50184d524b05d
C = cfb136abf320446887236da79a140f7d
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 25 | 15
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 12
Median | 26
Third Quartile | 30
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 128
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 02:14:32.846454 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:33.850239 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:34.877746 1782 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:35.878860 1783 meta_cache.cc:302] tablet 967cdb765da9410f89920ace401c5f51: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:36.883563 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:37.917927 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:38.972429 1782 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica e834766ed7b5410baf6fa83f5dfc43b9 (127.0.215.68:34223) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.68:34223: connect: Connection refused (error 111)
W20240503 02:14:40.006280 1783 meta_cache.cc:302] tablet cb3cca27ce284a7882131479b745e61e: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
W20240503 02:14:41.019795 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:42.046927 1785 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
W20240503 02:14:43.053696 1785 meta_cache.cc:302] tablet a001804c08de4aa9a989fd399bcbab85: replica b9327ad84d3f465a91f50184d524b05d (127.0.215.69:42309) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.69:42309: connect: Connection refused (error 111)
************************ BEGIN STACKS **************************
W20240503 02:14:44.055796 1783 meta_cache.cc:302] tablet ef979e6ffc7242bea2ca48b384720b40: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
[New LWP 864]
[New LWP 865]
[New LWP 866]
[New LWP 867]
[New LWP 868]
[New LWP 869]
[New LWP 870]
[New LWP 1750]
[New LWP 1751]
[New LWP 1752]
[New LWP 1753]
[New LWP 1782]
[New LWP 1783]
[New LWP 1784]
[New LWP 1785]
[New LWP 2230]
[New LWP 2231]
[New LWP 13153]
[New LWP 13257]
[New LWP 13269]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7feee205c380 (LWP 861) "raft_consensus_" 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7feee0037700 (LWP 864) "raft_consensus_" 0x00007feeee71632a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7feede65a700 (LWP 865) "kernel-watcher-" 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7feedde59700 (LWP 866) "rpc reactor-866" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7feedd658700 (LWP 867) "rpc reactor-867" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7feedce57700 (LWP 868) "rpc reactor-868" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7feedc656700 (LWP 869) "rpc reactor-869" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7feedbe55700 (LWP 870) "rpc reactor-870" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7feedb654700 (LWP 1750) "rpc reactor-175" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7feedac4c700 (LWP 1751) "rpc reactor-175" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7feeda44b700 (LWP 1752) "rpc reactor-175" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7feed9c4a700 (LWP 1753) "rpc reactor-175" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7feed8447700 (LWP 1782) "rpc reactor-178" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7feed7c46700 (LWP 1783) "rpc reactor-178" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7feed7445700 (LWP 1784) "rpc reactor-178" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7feed6c44700 (LWP 1785) "rpc reactor-178" 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7feed6443700 (LWP 2230) "raft_consensus_" 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
18 Thread 0x7feed5c42700 (LWP 2231) "raft_consensus_" 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
19 Thread 0x7feed4c40700 (LWP 13153) "raft_consensus_" 0x00007feeee711ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
20 Thread 0x7feed8c48700 (LWP 13257) "client-negotiat" 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
21 Thread 0x7feed443f700 (LWP 13269) "client-negotiat" 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 21 (Thread 0x7feed443f700 (LWP 13269)):
#0 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeee1d09d4 in kudu::ConditionVariable::WaitFor (this=0x7feed443e690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007feeee31e8d1 in kudu::ThreadPool::DispatchThread (this=0x55919c560f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007feeee31f6ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55919c952858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007feeee32103b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c952858) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c952800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 20 (Thread 0x7feed8c48700 (LWP 13257)):
#0 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeee1d09d4 in kudu::ConditionVariable::WaitFor (this=0x7feed8c47690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007feeee31e8d1 in kudu::ThreadPool::DispatchThread (this=0x55919c6aef00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007feeee31f6ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55919ca48558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007feeee32103b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919ca48558) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919ca48500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 19 (Thread 0x7feed4c40700 (LWP 13153)):
#0 0x00007feeee711ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeee1d0665 in kudu::ConditionVariable::Wait (this=0x55919c745a70) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:55
#2 0x00007feeeef5a4c1 in kudu::client::KuduSession::Data::Flush (this=0x55919c7459e0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/session-internal.cc:257
#3 0x00007feeeee64638 in kudu::client::KuduSession::Flush (this=0x55919c7a23e0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/client.cc:1524
#4 0x00007feeef78b7e3 in kudu::TestWorkload::WriteThread (this=0x7ffdd9d8cb10) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/test_workload.cc:204
#5 0x00007feeef795b8f in std::__invoke_impl<void, void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__f=@0x55919c88e2b0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7feeef78ad88 <kudu::TestWorkload::WriteThread()>, __t=@0x55919c88e2a8: 0x7ffdd9d8cb10) at /usr/include/c++/7/bits/invoke.h:73
#6 0x00007feeef795436 in std::__invoke<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__fn=@0x55919c88e2b0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7feeef78ad88 <kudu::TestWorkload::WriteThread()>) at /usr/include/c++/7/bits/invoke.h:95
#7 0x00007feeef796b07 in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::_M_invoke<0ul, 1ul> (this=0x55919c88e2a8) at /usr/include/c++/7/thread:234
#8 0x00007feeef796a6d in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::operator() (this=0x55919c88e2a8) at /usr/include/c++/7/thread:243
#9 0x00007feeef7969e2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> > >::_M_run (this=0x55919c88e2a0) at /usr/include/c++/7/thread:186
#10 0x00007feeed2d96df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 18 (Thread 0x7feed5c42700 (LWP 2231)):
#0 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeed898282 in base::SleepForNanoseconds (nanoseconds=1000000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007feeee2aea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055919a6f09cc in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55919c6d1358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:215
#4 0x000055919a6f3299 in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055919a6f2ef9 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055919a6f3d36 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55919c6d1358) at /usr/include/c++/7/thread:234
#7 0x000055919a6f3ce3 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55919c6d1358) at /usr/include/c++/7/thread:243
#8 0x000055919a6f3ca2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55919c6d1350) at /usr/include/c++/7/thread:186
#9 0x00007feeed2d96df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 17 (Thread 0x7feed6443700 (LWP 2230)):
#0 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeed898282 in base::SleepForNanoseconds (nanoseconds=250000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007feeee2aea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055919a6f052c in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55919c6d1178) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:196
#4 0x000055919a6f312f in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055919a6f2df6 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055919a6f3d62 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55919c6d1178) at /usr/include/c++/7/thread:234
#7 0x000055919a6f3d07 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55919c6d1178) at /usr/include/c++/7/thread:243
#8 0x000055919a6f3cc2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55919c6d1170) at /usr/include/c++/7/thread:186
#9 0x00007feeed2d96df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7feed6c44700 (LWP 1785)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c654880, timeout=0.045434149159063963) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c654880, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b5560, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b5558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c74a658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c74a658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c74a600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7feed7445700 (LWP 1784)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c654400, timeout=0.066521499159080122) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c654400, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b53a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b5398) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c74a558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c74a558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c74a500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7feed7c46700 (LWP 1783)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c653f80, timeout=0.06610011715906694) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c653f80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b51e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b51d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0e58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0e00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7feed8447700 (LWP 1782)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c6c2d80, timeout=0.045832490159227746) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c6c2d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b5020, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b5018) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0f58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0f58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7feed9c4a700 (LWP 1753)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c653b00, timeout=0.099098629159016127) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c653b00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4e60, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0358) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0300) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7feeda44b700 (LWP 1752)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c653680, timeout=0.099258816159135677) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c653680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4ca0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4c98) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0958) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0958) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7feedac4c700 (LWP 1751)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c6c3680, timeout=0.098480762158942525) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c6c3680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4ae0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4ad8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7feedb654700 (LWP 1750)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c6c3200, timeout=0.099560113158986496) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c6c3200, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4920, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4918) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0758) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0700) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7feedbe55700 (LWP 870)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c652d80, timeout=0.099938146159047392) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c652d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4220, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4218) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c1458) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c1458) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c1400) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7feedc656700 (LWP 869)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c652900, timeout=0.099096027159021105) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c652900, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b43e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b43d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c1058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c1058) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c1000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7feedce57700 (LWP 868)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c652480, timeout=0.099140860159195654) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c652480, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4760, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0858) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7feedd658700 (LWP 867)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c652000, timeout=0.099927019159167685) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c652000, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b4060, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c1258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c1258) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c1200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7feedde59700 (LWP 866)):
#0 0x00007feeec996a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007feee8199de0 in epoll_poll (loop=0x55919c515600, timeout=0.099191451159185817) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007feee819d49e in ev_run (loop=0x55919c515600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007feeee307ba3 in ev::loop_ref::run (this=0x55919c5b45a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007feeeab1c31d in kudu::rpc::ReactorThread::RunThread (this=0x55919c5b4598) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007feeeab18c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55919c4c0c58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007feeeab20934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c0c58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c0c00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7feede65a700 (LWP 865)):
#0 0x00007feeee711fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeee1d07eb in kudu::ConditionVariable::WaitUntil (this=0x55919c4a92d8, until=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:87
#2 0x00007feeef2a8fde in kudu::CountDownLatch::WaitUntil (this=0x55919c4a92a0, when=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:87
#3 0x00007feeef2a9088 in kudu::CountDownLatch::WaitFor (this=0x55919c4a92a0, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:97
#4 0x00007feeee268166 in kudu::KernelStackWatchdog::RunThread (this=0x55919c4a9200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007feeee267375 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x55919c4c1158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007feeee268aca in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007feeef59b856 in std::function<void ()>::operator()() const (this=0x55919c4c1158) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007feeee30e807 in kudu::Thread::SuperviseThread (arg=0x55919c4c1100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7feee0037700 (LWP 864)):
#0 0x00007feeee71632a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeee3057b5 in kudu::Subprocess::DoWait (this=0x7feee0036380, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:817
#2 0x00007feeee302be8 in kudu::Subprocess::Wait (this=0x7feee0036380, wait_status=0x0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:544
#3 0x00007feeee255b7c in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:258
#4 0x00007feeee2556ab in kudu::PstackWatcher::RunGdbStackDump (pid=861, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:218
#5 0x00007feeee254f09 in kudu::PstackWatcher::DumpPidStacks (pid=861, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:169
#6 0x00007feeee254e82 in kudu::PstackWatcher::DumpStacks (flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:162
#7 0x00007feeef646087 in kudu::<lambda()>::operator()(void) const (__closure=0x55919c5279f8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:60
#8 0x00007feeef646888 in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#9 0x00007feeef6465dc in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#10 0x00007feeef646e6e in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55919c5279f8) at /usr/include/c++/7/thread:234
#11 0x00007feeef646e13 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x55919c5279f8) at /usr/include/c++/7/thread:243
#12 0x00007feeef646dce in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x55919c5279f0) at /usr/include/c++/7/thread:186
#13 0x00007feeed2d96df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007feeee70b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007feeec99671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7feee205c380 (LWP 861)):
#0 0x00007feeee715d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007feeed898282 in base::SleepForNanoseconds (nanoseconds=1000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007feeee2aea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055919a6f18e1 in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody (this=0x55919c515a80) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:229
#4 0x00007feeee4b10ed in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7feeee4b282b "the test body", method=<optimized out>, object=0x55919c515a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#5 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x55919c515a80, method=<optimized out>, location=location@entry=0x7feeee4b282b "the test body") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#6 0x00007feeee4a5bdb in testing::Test::Run (this=0x55919c515a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#7 0x00007feeee4a5d9d in testing::TestInfo::Run (this=0x55919c4a8240) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#8 0x00007feeee4a6377 in testing::TestSuite::Run (this=0x55919c4a8480) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#9 0x00007feeee4a677c in testing::internal::UnitTestImpl::RunAllTests (this=<optimized out>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870
#10 0x00007feeee4b160d in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7feeee4b43d8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x55919c560280) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#11 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x55919c560280, method=<optimized out>, location=location@entry=0x7feeee4b43d8 "auxiliary test code (environments or event listeners)") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#12 0x00007feeee4a5e63 in testing::UnitTest::Run (this=0x7feeee6c51e0 <testing::UnitTest::GetInstance()::instance>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444
#13 0x00007feeef6471de in RUN_ALL_TESTS () at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#14 0x00007feeef64653d in main (argc=1, argv=0x7ffdd9d8d198) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:109
#15 0x00007feeec896bf7 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#16 0x000055919a6efe4a in _start ()
W20240503 02:15:14.855896 1783 meta_cache.cc:302] tablet daf20382e45943ca955bf509d5724f1d: replica cfb136abf320446887236da79a140f7d (127.0.215.70:37631) has failed: Network error: Client connection negotiation failed: client connection to 127.0.215.70:37631: connect: Connection refused (error 111)
************************* END STACKS ***************************
F20240503 02:15:14.924415 864 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1714702514 (unix time) try "date -d @1714702514" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e80000035d) received by PID 861 (TID 0x7feee0037700) from PID 861; stack trace: ***
@ 0x7feeee716980 (unknown) at ??:0
@ 0x7feeec8b3fb7 gsignal at ??:0
@ 0x7feeec8b5921 abort at ??:0
@ 0x7feeed5b8dcd google::LogMessage::Fail() at ??:0
@ 0x7feeed5bcb93 google::LogMessage::SendToLog() at ??:0
@ 0x7feeed5b87cc google::LogMessage::Flush() at ??:0
@ 0x7feeed5b9f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7feeef6461c4 _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at ??:0
@ 0x7feeef646888 _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at ??:0
@ 0x7feeef6465dc _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at ??:0
@ 0x7feeef646e6e _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at ??:0
@ 0x7feeef646e13 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at ??:0
@ 0x7feeef646dce _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at ??:0
@ 0x7feeed2d96df (unknown) at ??:0
@ 0x7feeee70b6db start_thread at ??:0
@ 0x7feeec99671f clone at ??:0