[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from RaftConsensusStressITest
[ RUN ] RaftConsensusStressITest.RemoveReplaceInCycle
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240503 13:58:35.749857 396 test_util.cc:268] Using random seed: 497658319
I20240503 13:58:35.756538 396 ts_itest-base.cc:112] Starting cluster with:
I20240503 13:58:35.756603 396 ts_itest-base.cc:113] --------------
I20240503 13:58:35.756615 396 ts_itest-base.cc:114] 6 tablet servers
I20240503 13:58:35.756628 396 ts_itest-base.cc:115] 3 replicas per TS
I20240503 13:58:35.756639 396 ts_itest-base.cc:116] --------------
2024-05-03T13:58:35Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-03T13:58:35Z Disabled control of system clock
I20240503 13:58:35.765741 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.99.62:46163
--webserver_interface=127.0.99.62
--webserver_port=0
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.99.62:46163
--minloglevel=2
--raft_prepare_replacement_before_eviction=true
--max_create_tablets_per_ts=40 with env {}
I20240503 13:58:35.941422 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 408
I20240503 13:58:35.941543 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240503 13:58:36.071631 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.1:0
--local_ip_for_outbound_sockets=127.0.99.1
--webserver_interface=127.0.99.1
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:36.268169 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 499
I20240503 13:58:36.268276 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240503 13:58:36.270663 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.2:0
--local_ip_for_outbound_sockets=127.0.99.2
--webserver_interface=127.0.99.2
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:36.457253 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 630
I20240503 13:58:36.457378 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240503 13:58:36.459616 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.3:0
--local_ip_for_outbound_sockets=127.0.99.3
--webserver_interface=127.0.99.3
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:36.655829 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 761
I20240503 13:58:36.655944 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240503 13:58:36.658094 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.4:0
--local_ip_for_outbound_sockets=127.0.99.4
--webserver_interface=127.0.99.4
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:36.854025 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 892
I20240503 13:58:36.854168 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240503 13:58:36.856200 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.5:0
--local_ip_for_outbound_sockets=127.0.99.5
--webserver_interface=127.0.99.5
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:37.052906 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 1023
I20240503 13:58:37.053043 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240503 13:58:37.055766 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-task2UEcZA/build/debug/bin/kudu
/tmp/dist-test-task2UEcZA/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.6:0
--local_ip_for_outbound_sockets=127.0.99.6
--webserver_interface=127.0.99.6
--webserver_port=0
--tserver_master_addrs=127.0.99.62:46163
--builtin_ntp_servers=127.0.99.20:38621
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240503 13:58:37.242009 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-task2UEcZA/build/debug/bin/kudu as pid 1154
I20240503 13:58:37.242126 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240503 13:58:37.257249 396 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240503 13:58:37.365393 396 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240503 13:58:37.367167 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 0bbb4d513d5443939e367f97daca7f0b to finish bootstrapping
I20240503 13:58:37.373559 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver a6c0f883cca44ef1a6478a19dcd5adb7 to finish bootstrapping
I20240503 13:58:37.379168 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 0c8d9fe05a8240db8587ee42048ab72a to finish bootstrapping
I20240503 13:58:37.385093 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 7a14f28360a64e8895fa011fae4f0781 to finish bootstrapping
I20240503 13:58:37.390825 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 306379e0d3b443c7bfb91a362d8724f2 to finish bootstrapping
I20240503 13:58:37.395781 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 0f8fa174bd444404a61b949964a72863 to finish bootstrapping
I20240503 13:58:37.400530 396 test_util.cc:268] Using random seed: 499309005
F20240503 13:58:47.438215 1953 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.queue_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.queue_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_cpu_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_cpu_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_wall_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_wall_time_us) (tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.queue_time_us, tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.queue_time_us) (tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.run_cpu_time_us, tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.run_cpu_time_us) (tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.run_wall_time_us, tablet-download-pool-2fcd20fbe4b543e781f0909ad48bf55c.run_wall_time_us) (tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.queue_time_us, tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.queue_time_us) (tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.run_cpu_time_us, tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.run_cpu_time_us) (tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.run_wall_time_us, tablet-download-pool-4aa996f2e85b4ecaa10ec1599cc61493.run_wall_time_us) (tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.queue_time_us, tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.queue_time_us) (tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.run_cpu_time_us, tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.run_cpu_time_us) (tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.run_wall_time_us, tablet-download-pool-6544f9e0bb184c7883b08b5ace6b7367.run_wall_time_us) (tablet-download-pool-6c9123c803274a3c85ec817f268783f7.queue_time_us, tablet-download-pool-6c9123c803274a3c85ec817f268783f7.queue_time_us) (tablet-download-pool-6c9123c803274a3c85ec817f268783f7.run_cpu_time_us, tablet-download-pool-6c9123c803274a3c85ec817f268783f7.run_cpu_time_us) (tablet-download-pool-6c9123c803274a3c85ec817f268783f7.run_wall_time_us, tablet-download-pool-6c9123c803274a3c85ec817f268783f7.run_wall_time_us) (tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.queue_time_us, tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.queue_time_us) (tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.run_cpu_time_us, tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.run_cpu_time_us) (tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.run_wall_time_us, tablet-download-pool-73e27245f699418b9ef9d309f89a59a8.run_wall_time_us) (tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.queue_time_us, tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.queue_time_us) (tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.run_cpu_time_us, tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.run_cpu_time_us) (tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.run_wall_time_us, tablet-download-pool-79b62ba9acd240e0b42c7f3bf572efdf.run_wall_time_us) (tablet-download-pool-8b1022c88a594f44bf75972364400fc2.queue_time_us, tablet-download-pool-8b1022c88a594f44bf75972364400fc2.queue_time_us) (tablet-download-pool-8b1022c88a594f44bf75972364400fc2.run_cpu_time_us, tablet-download-pool-8b1022c88a594f44bf75972364400fc2.run_cpu_time_us) (tablet-download-pool-8b1022c88a594f44bf75972364400fc2.run_wall_time_us, tablet-download-pool-8b1022c88a594f44bf75972364400fc2.run_wall_time_us) (tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.queue_time_us, tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.queue_time_us) (tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.run_cpu_time_us, tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.run_cpu_time_us) (tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.run_wall_time_us, tablet-download-pool-a9e628012ef24e0ca2b92831feab474e.run_wall_time_us) (tablet-download-pool-b9a111d617314f8daea992179a470839.queue_time_us, tablet-download-pool-b9a111d617314f8daea992179a470839.queue_time_us) (tablet-download-pool-b9a111d617314f8daea992179a470839.run_cpu_time_us, tablet-download-pool-b9a111d617314f8daea992179a470839.run_cpu_time_us) (tablet-download-pool-b9a111d617314f8daea992179a470839.run_wall_time_us, tablet-download-pool-b9a111d617314f8daea992179a470839.run_wall_time_us) (tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.queue_time_us, tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.queue_time_us) (tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.run_cpu_time_us, tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.run_cpu_time_us) (tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.run_wall_time_us, tablet-download-pool-bab2aa2b84b547f8b3ccf121b0778654.run_wall_time_us) (tablet-download-pool-bd476829e7ed46cbbf090997315dfded.queue_time_us, tablet-download-pool-bd476829e7ed46cbbf090997315dfded.queue_time_us) (tablet-download-pool-bd476829e7ed46cbbf090997315dfded.run_cpu_time_us, tablet-download-pool-bd476829e7ed46cbbf090997315dfded.run_cpu_time_us) (tablet-download-pool-bd476829e7ed46cbbf090997315dfded.run_wall_time_us, tablet-download-pool-bd476829e7ed46cbbf090997315dfded.run_wall_time_us) (tablet-download-pool-c2f3909abb4d4246ac57ed70f83c1224.queue_time_us, tablet-download-pool-c2f3909abb4d4246ac57ed70f83c1224.queue_time_us) (tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.queue_time_us, tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.queue_time_us) (tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.run_cpu_time_us, tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.run_cpu_time_us) (tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.run_wall_time_us, tablet-download-pool-d169c28b3ed145a8970d30cd78fad484.run_wall_time_us) (tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.queue_time_us, tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.queue_time_us) (tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.run_cpu_time_us, tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.run_cpu_time_us) (tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.run_wall_time_us, tablet-download-pool-d6dc919eda9a42898b35c94404fc4752.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f8bd6509dcd google::LogMessage::Fail() at ??:0
@ 0x7f8bd650db93 google::LogMessage::SendToLog() at ??:0
@ 0x7f8bd65097cc google::LogMessage::Flush() at ??:0
@ 0x7f8bd650af59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f8bd7973044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f8bd7957f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f8bd7955efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f8bda50877a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f8bda511a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f8bda59276f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f8bda5a64f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f8bda591226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f8bda5a0c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55fd324b575c std::function<>::operator()() at ??:0
@ 0x7f8bd795ae06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f8bd795b6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f8bd795d03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55fd324b575c std::function<>::operator()() at ??:0
@ 0x7f8bd794a807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f8bd81886db start_thread at ??:0
@ 0x7f8bd56c071f clone at ??:0
W20240503 13:58:48.056071 1319 meta_cache.cc:302] tablet e6688e1f618941218527450649eceeae: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 9 | 0
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | HEALTHY | <none> | 0 | 0
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | HEALTHY | <none> | 12 | 0
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | HEALTHY | <none> | 9 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
local_ip_for_outbound_sockets | 127.0.99.4 | experimental | 127.0.99.4:44245
local_ip_for_outbound_sockets | 127.0.99.5 | experimental | 127.0.99.5:45347
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:38763
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.0.99.4:44245
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.0.99.5:45347
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:38763
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+-----------------------------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
time_source | builtin | 127.0.99.3:38521, 127.0.99.6:38763, 127.0.99.5:45347, and 2 other server(s)
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.6:38763, tserver@127.0.99.5:45347, and 3 other server(s)
Tablet Summary
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | -1 | Yes
B | [config not available] | | |
C | A B C* | 2 | -1 | Yes
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): RUNNING
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C D~ | 2 | 62 | Yes
B | A B* C | 2 | 61 | Yes
C | A B* C | 2 | 61 | Yes
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 64 | Yes
B | A* B C | 1 | 64 | Yes
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D~ | 2 | 64 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 61 | Yes
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 61 | Yes
B | A B* C | 1 | 61 | Yes
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 68 | Yes
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D~ | 2 | 57 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 61 | Yes
B | A* B C | 1 | 61 | Yes
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D~ | 2 | 65 | No
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D~ | 2 | 62 | Yes
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 68 | Yes
B | A B* C | 1 | 68 | Yes
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 58 | Yes
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 65 | Yes
B | A B* C | 1 | 65 | Yes
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 65 | Yes
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 64 | Yes
B | A* B C | 2 | 64 | Yes
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D | 2 | 66 | No
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 57 | Yes
B | A B* C | 1 | 57 | Yes
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C D~ | 2 | 61 | Yes
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 59 | Yes
B | A* B C | 1 | 59 | Yes
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C D | 1 | 63 | Yes
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 65 | Yes
B | A B* C | 1 | 65 | Yes
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 62 | Yes
B | A* B C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A B C | 2 | 4 | Yes
B | A B* C | 3 | 4 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+--------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | CONSENSUS_MISMATCH | 40 | 9 | 0 | 30 | 1
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 20
Median | 21
Third Quartile | 21
Maximum | 31
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
-------+----------------------------------+------------------+---------------
Big | 0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | 31
Small | 0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | 10
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 123
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
F20240503 13:58:51.490864 2440 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_cpu_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_cpu_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_wall_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_wall_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us) (tablet-download-pool-15cbb7c04da949e9a672cedee7453915.queue_time_us, tablet-download-pool-15cbb7c04da949e9a672cedee7453915.queue_time_us) (tablet-download-pool-15cbb7c04da949e9a672cedee7453915.run_cpu_time_us, tablet-download-pool-15cbb7c04da949e9a672cedee7453915.run_cpu_time_us) (tablet-download-pool-15cbb7c04da949e9a672cedee7453915.run_wall_time_us, tablet-download-pool-15cbb7c04da949e9a672cedee7453915.run_wall_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us) (tablet-download-pool-822ab45e683c403487615cd16e364fb9.queue_time_us, tablet-download-pool-822ab45e683c403487615cd16e364fb9.queue_time_us) (tablet-download-pool-822ab45e683c403487615cd16e364fb9.run_cpu_time_us, tablet-download-pool-822ab45e683c403487615cd16e364fb9.run_cpu_time_us) (tablet-download-pool-822ab45e683c403487615cd16e364fb9.run_wall_time_us, tablet-download-pool-822ab45e683c403487615cd16e364fb9.run_wall_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us) (tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.queue_time_us, tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.queue_time_us) (tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.run_cpu_time_us, tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.run_cpu_time_us) (tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.run_wall_time_us, tablet-download-pool-b6e14b0dfe504cacba1308674bbbac8b.run_wall_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f41e478fdcd google::LogMessage::Fail() at ??:0
@ 0x7f41e4793b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f41e478f7cc google::LogMessage::Flush() at ??:0
@ 0x7f41e4790f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f41e5bf9044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f41e5bddf98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f41e5bdbefc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f41e878e77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f41e8797a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f41e881876f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f41e882c4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f41e8817226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f41e8826c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55bed3c3b75c std::function<>::operator()() at ??:0
@ 0x7f41e5be0e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f41e5be16ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f41e5be303b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55bed3c3b75c std::function<>::operator()() at ??:0
@ 0x7f41e5bd0807 kudu::Thread::SuperviseThread() at ??:0
W20240503 13:58:51.587675 1319 meta_cache.cc:302] tablet 796003537eee4ff185721db657759bf5: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
@ 0x7f41e640e6db start_thread at ??:0
@ 0x7f41e394671f clone at ??:0
W20240503 13:58:51.812987 1319 connection.cc:535] client connection to 127.0.99.4:44245 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240503 13:58:52.591277 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:58:53.659354 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:58:54.673956 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:58:55.739686 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:58:56.777467 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:58:57.817572 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:58:58.846639 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:58:59.954617 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:01.004907 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:02.264961 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:03.296191 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:04.538396 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:05.684094 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:06.699292 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:07.747756 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:08.840637 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:09.966751 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:11.126456 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:12.126920 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:13.352279 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:14.612061 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:15.906405 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:17.245260 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:18.613716 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:20.017505 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:21.461606 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:22.946595 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:23.956559 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:25.233529 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:26.280256 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:27.602751 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:28.686654 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:30.058059 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:31.177127 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:32.590831 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:33.746287 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:35.208227 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:36.398528 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:37.906253 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:39.137643 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:40.684167 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:41.944532 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:43.546839 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:44.842190 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:46.484440 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:47.821426 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:48.827818 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:49.843218 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:50.871306 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:51.554754 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 6544f9e0bb184c7883b08b5ace6b7367 after 1 attempt(s): Failed to write to server: 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): Write RPC to 127.0.99.5:45347 timed out after 59.975s (SENT)
W20240503 13:59:51.871726 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
F20240503 13:59:52.606451 5046 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_cpu_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_cpu_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_wall_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.run_wall_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us) (tablet-download-pool-1540e24763a94b189f5bd3223692ddb1.queue_time_us, tablet-download-pool-1540e24763a94b189f5bd3223692ddb1.queue_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_cpu_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_cpu_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_wall_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_wall_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.queue_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.queue_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.run_cpu_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.run_cpu_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.run_wall_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.run_wall_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.queue_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.queue_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_cpu_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_cpu_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_wall_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_wall_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fd0fadb6dcd google::LogMessage::Fail() at ??:0
@ 0x7fd0fadbab93 google::LogMessage::SendToLog() at ??:0
@ 0x7fd0fadb67cc google::LogMessage::Flush() at ??:0
@ 0x7fd0fadb7f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fd0fc220044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fd0fc204f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fd0fc202efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fd0fedb577a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fd0fedbea45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fd0fee3f76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fd0fee534f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fd0fee3e226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fd0fee4dc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x558b8063d75c std::function<>::operator()() at ??:0
@ 0x7fd0fc207e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fd0fc2086ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fd0fc20a03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x558b8063d75c std::function<>::operator()() at ??:0
@ 0x7fd0fc1f7807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fd0fca356db start_thread at ??:0
@ 0x7fd0f9f6d71f clone at ??:0
W20240503 13:59:52.850360 1319 connection.cc:535] client connection to 127.0.99.5:45347 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240503 13:59:52.871835 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 13:59:53.876700 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
F20240503 13:59:53.978940 5189 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us, tablet-download-pool-02112e93b6da4eeeb5178152c363ff2b.queue_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.queue_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_cpu_time_us) (tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us, tablet-download-pool-08b88ec93e224442a0434ecad6b1ef95.run_wall_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.queue_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_cpu_time_us) (tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us, tablet-download-pool-0cce1500f7be43b08284a85dda397c3d.run_wall_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.queue_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_cpu_time_us) (tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us, tablet-download-pool-186a7246498b427f8cef0a351470988c.run_wall_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.queue_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.queue_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_cpu_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_cpu_time_us) (tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_wall_time_us, tablet-download-pool-1a0879946ad04ea8a5a694e11c81b954.run_wall_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.queue_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_cpu_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_cpu_time_us) (tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_wall_time_us, tablet-download-pool-2c7bd903ec354fb9bc26dd823bc8d8c6.run_wall_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.queue_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_cpu_time_us) (tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us, tablet-download-pool-3e6afe59671548ed938a37a8fe1b89e2.run_wall_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.queue_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.queue_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.run_cpu_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.run_cpu_time_us) (tablet-download-pool-5ba32205508041549e61e34d8707f689.run_wall_time_us, tablet-download-pool-5ba32205508041549e61e34d8707f689.run_wall_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.queue_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_cpu_time_us) (tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us, tablet-download-pool-5e804571286d4bfb89ff84e113d8e456.run_wall_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.queue_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.queue_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_cpu_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_cpu_time_us) (tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_wall_time_us, tablet-download-pool-8e597f070f604749b5b9672dd290d1d4.run_wall_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.queue_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_cpu_time_us) (tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us, tablet-download-pool-a0a211f6dde14ab3b591b95d454ddba4.run_wall_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.queue_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_cpu_time_us) (tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us, tablet-download-pool-b9b123560ae84a42a72ddf9f23c33231.run_wall_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.queue_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_cpu_time_us) (tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us, tablet-download-pool-dcce7926af8f4c97873c5a1d8882835b.run_wall_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.queue_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_cpu_time_us) (tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us, tablet-download-pool-e09530df8fc04a6d8c9875845c4e7aa2.run_wall_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.queue_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_cpu_time_us) (tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us, tablet-download-pool-e6688e1f618941218527450649eceeae.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f0fe4132dcd google::LogMessage::Fail() at ??:0
@ 0x7f0fe4136b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f0fe41327cc google::LogMessage::Flush() at ??:0
@ 0x7f0fe4133f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f0fe559c044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f0fe5580f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f0fe557eefc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f0fe813177a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f0fe813aa45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f0fe81bb76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f0fe81cf4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f0fe81ba226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f0fe81c9c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55742040175c std::function<>::operator()() at ??:0
@ 0x7f0fe5583e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f0fe55846ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f0fe558603b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55742040175c std::function<>::operator()() at ??:0
@ 0x7f0fe5573807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f0fe5db16db start_thread at ??:0
@ 0x7f0fe32e971f clone at ??:0
W20240503 13:59:54.556736 402 connection.cc:535] client connection to 127.0.99.6:38763 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240503 13:59:54.878631 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 13:59:55.881282 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:56.881649 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:57.884657 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:58.904667 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 13:59:59.909404 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:00.931622 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:01.935142 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:02.954231 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:03.975116 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:04.995672 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:06.037258 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:07.052501 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:08.056838 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:09.084765 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:10.099380 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:11.167591 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:12.177358 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:13.193084 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:14.210127 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:15.212009 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:16.247951 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:17.289083 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:18.315637 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:19.349049 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:20.389536 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:21.481374 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:22.483215 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:23.522132 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:24.540650 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:25.555109 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:26.567749 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:27.607324 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:28.616948 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:29.651046 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:30.676085 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:31.685122 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:32.691232 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:33.713634 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:34.735882 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:35.764297 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:36.779021 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:37.799284 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:38.825042 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:39.878377 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:40.945506 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:00:41.957365 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:42.998523 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:44.001542 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:45.089215 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:46.092420 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:47.096899 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:48.181892 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:49.192039 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:50.215263 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:51.234870 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:51.818718 1319 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 79b62ba9acd240e0b42c7f3bf572efdf after 343 attempt(s): Failed to write to server: (no server available): Write(tablet: 79b62ba9acd240e0b42c7f3bf572efdf, num_ops: 3, num_attempts: 343) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.2:36267, tserver@127.0.99.3:38521
Tablet Summary
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [NONVOTER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 180 | Yes
C | A B C* D~ | 1 | 180 | Yes
D | [config not available] | | |
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C D~ | 1 | 125 | Yes
B | A* B C D~ | 1 | 125 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C D~ | 1 | 180 | Yes
B | A B* C D~ | 1 | 180 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [NONVOTER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 177 | Yes
C | A B C* D~ | 2 | 177 | Yes
D | [config not available] | | |
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 183 | Yes
B | A B* C | 2 | 183 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [NONVOTER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 131 | Yes
B | A* B C D~ | 1 | 131 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C D~ | 2 | 122 | Yes
B | A* B C D~ | 2 | 122 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 173 | Yes
B | A B* C | 2 | 173 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 121 | Yes
B | A* B C | 1 | 121 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [NONVOTER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
D = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 126 | Yes
B | A* B C D~ | 1 | 126 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 177 | Yes
C | A B C* | 2 | 177 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 177 | Yes
C | A B C* | 1 | 177 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [NONVOTER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
D = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 2 | 122 | Yes
B | A* B C D~ | 2 | 122 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 179 | Yes
C | A B C* | 1 | 179 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [NONVOTER]
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
D = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 3 | 66 | Yes
B | A* B C D~ | 3 | 66 | Yes
C | [config not available] | | |
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 30
Third Quartile | 31
Maximum | 33
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 139
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:00:52.423584 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:53.429359 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:54.432060 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:00:55.456187 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:56.464277 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:57.495594 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:00:58.504827 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:00:59.506203 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:00.524179 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:01.532691 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:02.602233 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:03.669214 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:04.697109 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:05.705464 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:06.708870 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:07.734174 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:08.734454 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:09.736984 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:10.857430 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:12.030025 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:13.035342 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:14.050784 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:15.102046 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:16.181859 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:17.195144 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:18.198300 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:19.234319 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:20.271076 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:21.362179 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:22.433750 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:23.433833 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:24.434327 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:25.446280 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:26.479822 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:27.528430 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:28.593645 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:29.678876 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:30.778947 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:31.781276 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:33.032305 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:34.054280 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:35.056411 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:36.092026 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:37.112363 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:38.317292 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:39.393433 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:40.462533 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:41.569546 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:42.652541 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:43.926843 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:45.082367 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:46.199404 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:47.201712 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:48.202456 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:01:49.205763 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:50.212694 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:51.236135 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:52.268980 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:52.424428 1320 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 056f24bfc37a46249853000caf784fae after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.995s (SENT)
W20240503 14:01:53.274016 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:54.283898 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:01:55.287305 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:56.297075 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:57.300542 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:01:58.319171 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:01:59.322481 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:00.335228 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:01.362221 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:02.387099 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:03.388195 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:04.439667 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:05.456570 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:06.519315 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:07.536126 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:08.582643 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:09.590875 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:10.605109 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:11.610076 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:12.627746 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:13.637653 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:14.644995 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:15.652264 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:16.662732 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:17.663429 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:18.683717 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:19.684098 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:20.710135 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:21.738698 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:22.791971 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:23.807265 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:24.815631 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:25.842756 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:26.849387 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:27.895946 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:28.964269 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:29.971063 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:30.977185 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:31.996259 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:33.127683 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:34.140095 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:35.140353 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:36.163295 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:37.198778 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:38.216811 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:39.313519 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:40.321434 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:41.325322 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:42.350800 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:43.371130 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:44.385238 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:45.402844 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:46.443950 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:47.456737 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:48.526237 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:49.534060 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:50.557179 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:51.582412 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:52.618378 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:52.642848 1320 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet bd476829e7ed46cbbf090997315dfded after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 60.000s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.2:36267, tserver@127.0.99.3:38521
Tablet Summary
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:02:53.619288 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:02:54.629657 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:55.636250 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:56.636907 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:02:57.638053 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:02:58.638790 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:02:59.662168 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:00.666384 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:01.679415 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:02.711619 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:03.723126 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:04.725836 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:05.729935 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:06.736886 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:07.740304 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:08.741190 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:09.757597 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:10.762202 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:11.787901 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:12.808065 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:13.809772 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:14.833499 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:15.833640 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:16.863354 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:17.879387 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:18.917474 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:19.929523 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:20.949664 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:21.963236 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:22.998044 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:23.998446 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:24.999943 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:26.012957 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:27.026854 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:28.032955 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:29.051731 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:30.059998 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:31.062155 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:32.064314 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:33.097867 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:34.116281 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:35.186964 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:36.210048 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:37.242841 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:38.247072 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:39.249985 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:40.262989 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:41.263810 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:42.307142 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:43.384567 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:44.409384 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:45.432760 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:46.447877 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:47.594090 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:48.596995 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:49.599880 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:50.603605 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:51.621629 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:03:52.648134 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:53.225679 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 2c5d3fa85f394d8fb89e605108da457c after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.998s (SENT)
W20240503 14:03:53.648160 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:54.652925 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:03:55.661015 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:56.670197 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:03:57.683660 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:58.686316 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:03:59.696988 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:00.735538 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:01.765709 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:02.778029 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:03.791380 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:04.794445 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:05.802597 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:06.817623 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:07.817935 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:08.846612 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:09.916472 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:10.930655 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:11.964565 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:12.974586 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:13.980304 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:14.995805 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:15.999927 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:17.074560 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:18.170961 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:19.198513 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:20.206247 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:21.213840 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:22.323583 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:23.340711 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:24.340752 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:25.342640 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:26.346212 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:27.368770 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:28.369771 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:29.425422 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:30.507196 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:31.600145 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:32.617056 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:33.629093 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:34.697335 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:35.856128 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:36.920634 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:37.923394 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:39.006889 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:40.030733 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:41.035040 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:04:42.190392 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:43.195928 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:44.279937 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:45.351641 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:46.381615 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:04:47.385658 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:48.391750 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:49.500572 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:50.504674 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:51.525276 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:52.550525 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:53.574174 1320 meta_cache.cc:302] tablet bd476829e7ed46cbbf090997315dfded: replica 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521) has failed: Timed out: Write RPC to 127.0.99.3:38521 timed out after 59.995s (SENT)
W20240503 14:04:53.574267 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet bd476829e7ed46cbbf090997315dfded after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.995s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.3:38521, tserver@127.0.99.2:36267
Tablet Summary
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:04:54.579245 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:55.580080 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:56.584815 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:57.590521 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:04:58.591279 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:04:59.604527 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:00.667618 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:01.742149 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:02.754097 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:03.757313 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:04.830711 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:05.878206 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:06.895813 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:07.960286 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:08.976234 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:10.032670 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:11.127485 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:12.129951 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:13.221948 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:14.227962 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:15.228323 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:16.264214 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:17.331777 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:18.425673 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:19.541503 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:20.552903 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:21.613763 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:22.630448 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:05:23.631596 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:24.746627 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:25.752306 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:26.767428 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:27.800765 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:28.844285 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:29.904109 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:30.986230 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:32.079077 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:33.189716 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:34.320014 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:35.322072 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:36.334733 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:37.338557 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:38.391371 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:39.396458 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:40.501646 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:41.501654 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:05:42.511374 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:43.599210 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:44.637323 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:05:45.638222 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:46.802047 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:47.805881 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:48.814172 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:49.814527 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:50.817447 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:51.835229 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:05:52.842994 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:53.862107 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:54.071537 1320 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet c2f3909abb4d4246ac57ed70f83c1224 after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.995s (SENT)
W20240503 14:05:54.866448 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:55.868508 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:56.872294 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:57.875088 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:05:58.876515 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:05:59.878183 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:00.885051 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:01.897446 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:02.898273 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:03.922189 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:04.957800 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:06.011046 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:07.013458 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:08.020646 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:09.029886 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:10.034664 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:11.038457 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:12.041033 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:13.043211 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:14.080108 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:15.090176 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:16.117547 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:17.171092 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:18.257273 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:19.303277 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:20.308583 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:21.364989 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:22.368633 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:23.413513 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:24.533196 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:25.564839 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:26.574062 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:27.593927 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:28.599121 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:29.649140 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:30.722210 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:31.739745 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:32.748667 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:33.752346 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:34.878666 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:35.962478 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:37.015574 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:38.072523 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:39.081526 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:40.169471 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:41.192507 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:42.236544 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:43.247929 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:06:44.301724 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:45.383846 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:46.427140 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:47.431326 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:48.437572 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:06:49.483748 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:50.484373 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:51.493373 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:52.513234 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:53.541739 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:06:54.361502 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet a3da5f059c0746ca9311746f79d8494d after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.996s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.2:36267, tserver@127.0.99.3:38521
Tablet Summary
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:06:54.857595 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:55.859186 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:56.859458 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:57.867112 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:58.871016 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:06:59.876605 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:00.882297 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:01.890125 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:02.928323 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:03.936314 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:04.946372 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:05.966796 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:06.969441 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:07.991750 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:09.002799 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:10.005609 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:11.009788 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:12.050076 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:13.050662 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:14.066165 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:15.075171 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:16.083667 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:17.086230 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:18.130517 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:19.131400 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:20.131436 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:21.132261 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:22.172233 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:23.176379 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:24.191882 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:25.208257 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:26.288178 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:27.299933 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:28.322142 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:29.361382 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:30.433672 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:31.490664 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:32.493979 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:33.504379 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:34.546285 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:35.568532 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:36.582293 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:37.588972 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:38.593930 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:39.618922 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:40.692394 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:41.692710 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:42.699931 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:43.752025 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:44.794661 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:45.808539 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:46.816406 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:07:47.905772 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:48.921468 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:49.931430 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:50.933185 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:51.943727 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:52.961045 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:07:53.962981 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:54.855042 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet bd476829e7ed46cbbf090997315dfded after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.996s (SENT)
W20240503 14:07:55.188467 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:56.189533 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:57.190351 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:07:58.197233 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:07:59.200018 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:00.215181 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:01.228075 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:02.241472 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:03.243773 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:04.265010 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:05.267685 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:06.286249 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:07.315766 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:08.320520 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:09.340893 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:10.361507 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:11.367904 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:12.375180 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:13.375926 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:14.384047 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:15.394559 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:16.417194 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:17.419139 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:18.467669 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:19.481329 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:20.486949 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:21.507719 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:22.512555 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:23.521584 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:24.530648 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:25.559768 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:26.578785 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:27.632694 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:28.657249 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:29.657635 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:30.681950 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:31.691744 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:32.714286 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:33.717011 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:34.725808 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:35.758455 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:36.791692 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:37.813104 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:38.835276 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:39.843191 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:40.851342 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:41.939647 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:42.944056 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:43.999408 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:45.056336 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:08:46.057860 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:47.069346 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:48.073834 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:49.113466 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:50.181164 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:51.236553 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:52.244235 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:53.258316 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:54.285240 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:08:55.187953 1320 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 16110beee7af425abe6d44e84c0bae94 after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.995s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.3:38521, tserver@127.0.99.2:36267
Tablet Summary
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:08:55.776443 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:56.779284 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:57.781620 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:08:58.792003 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:08:59.797612 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:00.799580 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:01.819922 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:02.843760 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:03.863250 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:04.880652 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:05.886777 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:06.906368 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:07.949158 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:08.957989 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:09.963919 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:10.974411 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:11.988394 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:13.000341 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:14.037019 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:15.044112 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:16.044306 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:17.053943 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:18.058414 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:19.125067 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:20.216357 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:21.234767 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:22.242583 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:23.270093 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:24.351387 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:25.360795 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:26.367715 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:27.371001 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:28.377643 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:29.379729 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:30.389220 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:31.456352 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:32.530141 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:33.628522 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:34.740152 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:35.864514 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:37.006300 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:38.161842 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:39.166909 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:40.223958 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:09:41.228905 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:42.330191 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:43.341661 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:44.487654 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:45.491302 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:46.694664 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:47.696805 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:48.708729 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:49.720084 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:50.729148 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:51.729274 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:52.732496 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:53.737947 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:54.764155 1319 meta_cache.cc:302] tablet 71036e742ef54e57aa91ac0573e5aa52: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:09:55.772200 1320 meta_cache.cc:302] tablet 056f24bfc37a46249853000caf784fae: replica 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521) has failed: Timed out: Write RPC to 127.0.99.3:38521 timed out after 59.993s (SENT)
W20240503 14:09:55.772331 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 056f24bfc37a46249853000caf784fae after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.993s (SENT)
W20240503 14:09:56.776396 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:09:57.794373 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:58.796100 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:09:59.832612 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:00.835757 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:01.855197 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:02.860623 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:03.898591 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:04.928886 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:05.930641 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:06.945190 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:07.977957 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:09.028668 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:10.065785 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:11.090667 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:12.145284 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:13.172168 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:14.182521 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:15.182556 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:16.202473 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:10:17.217432 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:10:18.221508 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:19.232231 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:20.254920 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:21.256695 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:22.274734 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:23.297741 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:24.371668 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:25.394845 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:26.401018 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:27.406288 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:28.419462 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:29.425138 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:30.438421 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:31.495316 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:32.572225 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:33.653434 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:34.669715 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:35.697651 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:10:36.735584 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:37.782054 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:38.783841 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:39.826097 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:40.831853 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:41.915635 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:42.931716 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:43.964608 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:44.983374 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:46.038441 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:47.046422 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:48.158671 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:49.245888 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:50.253672 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:51.264261 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:52.268304 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:53.280174 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:54.300525 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:55.328594 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:56.108768 1320 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 2c5d3fa85f394d8fb89e605108da457c after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.990s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.3:38521, tserver@127.0.99.2:36267
Tablet Summary
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:10:56.741981 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:10:57.743310 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:10:58.745481 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:10:59.751673 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:00.755903 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:01.760677 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:02.767730 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:03.795115 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:04.810534 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:05.820574 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:06.823608 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:07.841909 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:08.856431 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:09.858048 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:10.879954 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:11.906497 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:12.940706 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:13.949745 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:14.951018 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:15.960434 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:16.980125 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:17.996523 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:19.005041 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:20.066057 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:21.116334 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:22.119067 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:23.146545 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:24.164709 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:25.209946 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:26.267086 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:27.283340 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:28.286890 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:29.296015 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:30.298676 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:31.322630 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:32.385334 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:33.399320 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:34.441812 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:35.449949 1319 meta_cache.cc:302] tablet 2fcd20fbe4b543e781f0909ad48bf55c: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:36.504784 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:37.519704 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:38.544008 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:39.602710 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:40.617462 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:41.740381 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:42.795128 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:43.795881 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:44.813974 1319 meta_cache.cc:302] tablet 4aa996f2e85b4ecaa10ec1599cc61493: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:45.886651 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:46.922710 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:47.926558 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:11:48.955983 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:11:49.957600 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:51.027576 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:52.041989 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:53.045521 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:54.051241 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:55.052610 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:56.079284 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:11:56.741048 1320 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 056f24bfc37a46249853000caf784fae after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.998s (SENT)
W20240503 14:11:57.081158 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:58.087525 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:11:59.092284 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:00.101987 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:01.103407 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:02.113740 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:03.114681 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:04.121824 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:05.138789 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:06.153378 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:07.157824 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:08.161476 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:09.163494 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:10.183462 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:11.187104 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:12.203081 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:13.240521 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:14.240672 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:15.271683 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:16.283303 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:17.285468 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:18.293632 1319 meta_cache.cc:302] tablet 73e27245f699418b9ef9d309f89a59a8: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:19.296033 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:20.380259 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:21.380704 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:22.385298 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:23.405503 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:24.429594 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:25.457979 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:26.475555 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:27.540107 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:28.552361 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:29.565798 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:30.572988 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:31.618505 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:32.699918 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:33.756587 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:34.760438 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:35.794112 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:36.816895 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:37.937521 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:38.989676 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:40.013394 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:41.035423 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:42.060520 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:43.067348 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:44.174814 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:45.206404 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:46.232566 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:47.293037 1319 meta_cache.cc:302] tablet f07cd034451d4811bd599db2e2934ae8: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:48.294222 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:49.321367 1319 meta_cache.cc:302] tablet 6c9123c803274a3c85ec817f268783f7: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:50.323937 1319 meta_cache.cc:302] tablet bab2aa2b84b547f8b3ccf121b0778654: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:51.359038 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111)
W20240503 14:12:52.374442 1319 meta_cache.cc:302] tablet b9a111d617314f8daea992179a470839: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:53.517812 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:54.530598 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111)
W20240503 14:12:55.553490 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:12:56.582331 1319 meta_cache.cc:302] tablet 6544f9e0bb184c7883b08b5ace6b7367: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:57.067701 1320 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet a3da5f059c0746ca9311746f79d8494d after 1 attempt(s): Failed to write to server: 0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): Write RPC to 127.0.99.3:38521 timed out after 59.998s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
7293673986ac4175a8338575b88c9b34 | 127.0.99.62:46163 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:38621 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
0c8d9fe05a8240db8587ee42048ab72a | 127.0.99.3:38521 | HEALTHY | <none> | 15 | 0
a6c0f883cca44ef1a6478a19dcd5adb7 | 127.0.99.2:36267 | HEALTHY | <none> | 11 | 0
0bbb4d513d5443939e367f97daca7f0b | 127.0.99.1:34857 | UNAVAILABLE | <none> | n/a | n/a
0f8fa174bd444404a61b949964a72863 | 127.0.99.6:38763 | UNAVAILABLE | <none> | n/a | n/a
306379e0d3b443c7bfb91a362d8724f2 | 127.0.99.5:45347 | UNAVAILABLE | <none> | n/a | n/a
7a14f28360a64e8895fa011fae4f0781 | 127.0.99.4:44245 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.1:34857: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.1:34857: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.6:38763: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.6:38763: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.5:45347: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:44245: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:38521, 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:36267
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:38521
never_fsync | true | unsafe,advanced | 127.0.99.3:38521, 127.0.99.2:36267
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:38521, 127.0.99.2:36267
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:38521, 127.0.99.2:36267
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_format | pb | hidden | 127.0.99.3:38521, 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:36267
server_dump_info_path | /tmp/dist-test-task2UEcZA/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714744715738423-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:38521
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:38521, 127.0.99.2:36267
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------
builtin_ntp_servers | 127.0.99.20:38621 | 127.0.99.3:38521, 127.0.99.2:36267
time_source | builtin | 127.0.99.3:38521, 127.0.99.2:36267
Version Summary
Version | Servers
-----------------+------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:46163, tserver@127.0.99.3:38521, tserver@127.0.99.2:36267
Tablet Summary
Tablet 1a0879946ad04ea8a5a694e11c81b954 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 67 | Yes
B | A* B C | 3 | 67 | Yes
C | [config not available] | | |
Tablet f07cd034451d4811bd599db2e2934ae8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0bbb4d513d5443939e367f97daca7f0b
C = 7a14f28360a64e8895fa011fae4f0781
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 2 | 64 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 71036e742ef54e57aa91ac0573e5aa52 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet 71036e742ef54e57aa91ac0573e5aa52: Tablet copy aborted
All reported replicas are:
A = 0c8d9fe05a8240db8587ee42048ab72a
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0f8fa174bd444404a61b949964a72863
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | A B C* D~ | 3 | 66 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b6e14b0dfe504cacba1308674bbbac8b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 184 | Yes
C | A B C* | 1 | 184 | Yes
Tablet 4aa996f2e85b4ecaa10ec1599cc61493 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e6688e1f618941218527450649eceeae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet bab2aa2b84b547f8b3ccf121b0778654 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 117 | Yes
D | A B C D~ | 2 | 117 | Yes
Tablet 08b88ec93e224442a0434ecad6b1ef95 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet 2c5d3fa85f394d8fb89e605108da457c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 4 | 124 | No
D | A B C* D | 4 | 124 | No
Tablet 8e597f070f604749b5b9672dd290d1d4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 182 | Yes
B | A B* C | 1 | 182 | Yes
C | [config not available] | | |
Tablet a3da5f059c0746ca9311746f79d8494d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 125 | No
D | A B C* D | 3 | 125 | No
Tablet 796003537eee4ff185721db657759bf5 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 181 | Yes
C | A B C* | 2 | 181 | Yes
Tablet 16110beee7af425abe6d44e84c0bae94 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 122 | No
D | A B C* D | 3 | 122 | No
Tablet 5ba32205508041549e61e34d8707f689 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 186 | Yes
B | A B* C | 2 | 186 | Yes
C | [config not available] | | |
Tablet d169c28b3ed145a8970d30cd78fad484 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 3e6afe59671548ed938a37a8fe1b89e2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 133 | Yes
B | A* B C | 1 | 133 | Yes
C | [config not available] | | |
Tablet bd476829e7ed46cbbf090997315dfded of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 2 | 128 | No
D | A B C* D | 2 | 128 | No
Tablet 5e804571286d4bfb89ff84e113d8e456 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 124 | Yes
B | A* B C | 2 | 124 | Yes
C | [config not available] | | |
Tablet c2f3909abb4d4246ac57ed70f83c1224 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 129 | No
D | A B C* D | 3 | 129 | No
Tablet 2c7bd903ec354fb9bc26dd823bc8d8c6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 0f8fa174bd444404a61b949964a72863
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 176 | Yes
B | A B* C | 2 | 176 | Yes
C | [config not available] | | |
Tablet 79b62ba9acd240e0b42c7f3bf572efdf of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e09530df8fc04a6d8c9875845c4e7aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet b9a111d617314f8daea992179a470839 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): not running [NONVOTER]
State: FAILED
Data state: TABLET_DATA_TOMBSTONED
Last status: Tablet Copy: Tombstoned tablet b9a111d617314f8daea992179a470839: Tablet copy aborted
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 3 | 121 | Yes
D | [config not available] | | |
Tablet 02112e93b6da4eeeb5178152c363ff2b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 127 | Yes
B | A* B C | 2 | 127 | Yes
C | [config not available] | | |
Tablet a9e628012ef24e0ca2b92831feab474e of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b9b123560ae84a42a72ddf9f23c33231 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 124 | Yes
B | A* B C | 1 | 124 | Yes
C | [config not available] | | |
Tablet 6c9123c803274a3c85ec817f268783f7 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet dcce7926af8f4c97873c5a1d8882835b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 130 | Yes
B | A* B C | 1 | 130 | Yes
C | [config not available] | | |
Tablet 8b1022c88a594f44bf75972364400fc2 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 186a7246498b427f8cef0a351470988c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 130 | Yes
B | A* B C | 2 | 130 | Yes
C | [config not available] | | |
Tablet 6544f9e0bb184c7883b08b5ace6b7367 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1540e24763a94b189f5bd3223692ddb1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 180 | Yes
C | A B C* | 2 | 180 | Yes
Tablet 6e97c5d09c76425888e674e5ee9c2eed of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 4 | 119 | Yes
D | A B C D~ | 4 | 119 | Yes
Tablet 15cbb7c04da949e9a672cedee7453915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 180 | Yes
C | A B C* | 1 | 180 | Yes
Tablet 056f24bfc37a46249853000caf784fae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0f8fa174bd444404a61b949964a72863 (127.0.99.6:38763): TS unavailable
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [NONVOTER]
All reported replicas are:
A = 306379e0d3b443c7bfb91a362d8724f2
B = 0f8fa174bd444404a61b949964a72863
C = 0c8d9fe05a8240db8587ee42048ab72a
D = a6c0f883cca44ef1a6478a19dcd5adb7
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 3 | 124 | No
D | A B C* D | 3 | 124 | No
Tablet 0cce1500f7be43b08284a85dda397c3d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 126 | Yes
B | A* B C | 2 | 126 | Yes
C | [config not available] | | |
Tablet 73e27245f699418b9ef9d309f89a59a8 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet a0a211f6dde14ab3b591b95d454ddba4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING [LEADER]
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
All reported replicas are:
A = a6c0f883cca44ef1a6478a19dcd5adb7
B = 0c8d9fe05a8240db8587ee42048ab72a
C = 306379e0d3b443c7bfb91a362d8724f2
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 127 | Yes
B | A* B C | 1 | 127 | Yes
C | [config not available] | | |
Tablet d6dc919eda9a42898b35c94404fc4752 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable [LEADER]
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 822ab45e683c403487615cd16e364fb9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
a6c0f883cca44ef1a6478a19dcd5adb7 (127.0.99.2:36267): RUNNING
0c8d9fe05a8240db8587ee42048ab72a (127.0.99.3:38521): RUNNING [LEADER]
All reported replicas are:
A = 0bbb4d513d5443939e367f97daca7f0b
B = a6c0f883cca44ef1a6478a19dcd5adb7
C = 0c8d9fe05a8240db8587ee42048ab72a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | 181 | Yes
C | A B C* | 1 | 181 | Yes
Tablet 2fcd20fbe4b543e781f0909ad48bf55c of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245): TS unavailable [LEADER]
306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347): TS unavailable
0bbb4d513d5443939e367f97daca7f0b (127.0.99.1:34857): TS unavailable
All reported replicas are:
A = 7a14f28360a64e8895fa011fae4f0781
B = 306379e0d3b443c7bfb91a362d8724f2
C = 0bbb4d513d5443939e367f97daca7f0b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 19 | 21
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 16
Median | 29
Third Quartile | 30
Maximum | 31
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240503 14:12:57.697474 1319 meta_cache.cc:302] tablet d169c28b3ed145a8970d30cd78fad484: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:58.699075 1319 meta_cache.cc:302] tablet 8b1022c88a594f44bf75972364400fc2: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:12:59.703670 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:13:00.709977 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:13:01.715402 1319 meta_cache.cc:302] tablet 6e97c5d09c76425888e674e5ee9c2eed: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:13:02.729835 1319 meta_cache.cc:302] tablet 79b62ba9acd240e0b42c7f3bf572efdf: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:13:03.741050 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
W20240503 14:13:04.782420 1319 meta_cache.cc:302] tablet d6dc919eda9a42898b35c94404fc4752: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
W20240503 14:13:05.786098 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 306379e0d3b443c7bfb91a362d8724f2 (127.0.99.5:45347) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:45347: connect: Connection refused (error 111)
************************ BEGIN STACKS **************************
[New LWP 399]
[New LWP 400]
[New LWP 401]
[New LWP 402]
[New LWP 403]
[New LWP 404]
[New LWP 405]
[New LWP 1285]
[New LWP 1286]
[New LWP 1287]
[New LWP 1288]
[New LWP 1318]
[New LWP 1319]
[New LWP 1320]
[New LWP 1321]
[New LWP 1845]
[New LWP 1846]
[New LWP 11737]
[New LWP 11746]
[New LWP 11845]
[New LWP 11861]
[New LWP 11862]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7fcde580d380 (LWP 396) "raft_consensus_" 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7fcde37e8700 (LWP 399) "raft_consensus_" 0x00007fcdf1ec732a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7fcde1bfd700 (LWP 400) "kernel-watcher-" 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7fcde13fc700 (LWP 401) "rpc reactor-401" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7fcde0bfb700 (LWP 402) "rpc reactor-402" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7fcde03fa700 (LWP 403) "rpc reactor-403" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7fcddfbf9700 (LWP 404) "rpc reactor-404" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7fcddf3f8700 (LWP 405) "rpc reactor-405" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7fcddebf7700 (LWP 1285) "rpc reactor-128" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7fcdde1ef700 (LWP 1286) "rpc reactor-128" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7fcddd9ee700 (LWP 1287) "rpc reactor-128" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7fcddd1ed700 (LWP 1288) "rpc reactor-128" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7fcddb9ea700 (LWP 1318) "rpc reactor-131" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7fcddb1e9700 (LWP 1319) "rpc reactor-131" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7fcdda9e8700 (LWP 1320) "rpc reactor-132" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7fcdda1e7700 (LWP 1321) "rpc reactor-132" 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7fcdd99e6700 (LWP 1845) "raft_consensus_" 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
18 Thread 0x7fcdd91e5700 (LWP 1846) "raft_consensus_" 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
19 Thread 0x7fcdd79e2700 (LWP 11737) "raft_consensus_" 0x00007fcdf1ec2ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
20 Thread 0x7fcdd89e4700 (LWP 11746) "client-negotiat" 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
21 Thread 0x7fcddc9ec700 (LWP 11845) "client-negotiat" 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
22 Thread 0x7fcddc1eb700 (LWP 11861) "client-negotiat" 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
23 Thread 0x7fcdd71e1700 (LWP 11862) "client-negotiat" 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 23 (Thread 0x7fcdd71e1700 (LWP 11862)):
#0 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf19819d4 in kudu::ConditionVariable::WaitFor (this=0x7fcdd71e0690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fcdf1acf8d1 in kudu::ThreadPool::DispatchThread (this=0x55c0e18f6f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fcdf1ad06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55c0e1b09158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fcdf1ad203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1b09158) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1b09100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 22 (Thread 0x7fcddc1eb700 (LWP 11861)):
#0 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf19819d4 in kudu::ConditionVariable::WaitFor (this=0x7fcddc1ea690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fcdf1acf8d1 in kudu::ThreadPool::DispatchThread (this=0x55c0e18f6f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fcdf1ad06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55c0e1b08058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fcdf1ad203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1b08058) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1b08000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 21 (Thread 0x7fcddc9ec700 (LWP 11845)):
#0 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf19819d4 in kudu::ConditionVariable::WaitFor (this=0x7fcddc9eb690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fcdf1acf8d1 in kudu::ThreadPool::DispatchThread (this=0x55c0e1a44f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fcdf1ad06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55c0e1b09058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fcdf1ad203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1b09058) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1b09000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 20 (Thread 0x7fcdd89e4700 (LWP 11746)):
#0 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf19819d4 in kudu::ConditionVariable::WaitFor (this=0x7fcdd89e3690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fcdf1acf8d1 in kudu::ThreadPool::DispatchThread (this=0x55c0e1a44f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fcdf1ad06ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55c0e1b09258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fcdf1ad203b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1b09258) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1b09200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 19 (Thread 0x7fcdd79e2700 (LWP 11737)):
#0 0x00007fcdf1ec2ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf1981665 in kudu::ConditionVariable::Wait (this=0x55c0e1e21cb0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:55
#2 0x00007fcdf270b4c1 in kudu::client::KuduSession::Data::Flush (this=0x55c0e1e21c20) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/session-internal.cc:257
#3 0x00007fcdf2615638 in kudu::client::KuduSession::Flush (this=0x55c0e1a18220) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/client.cc:1524
#4 0x00007fcdf2f3c7e3 in kudu::TestWorkload::WriteThread (this=0x7ffc7a825b60) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/test_workload.cc:204
#5 0x00007fcdf2f46b8f in std::__invoke_impl<void, void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__f=@0x55c0e1e952d0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7fcdf2f3bd88 <kudu::TestWorkload::WriteThread()>, __t=@0x55c0e1e952c8: 0x7ffc7a825b60) at /usr/include/c++/7/bits/invoke.h:73
#6 0x00007fcdf2f46436 in std::__invoke<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__fn=@0x55c0e1e952d0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7fcdf2f3bd88 <kudu::TestWorkload::WriteThread()>) at /usr/include/c++/7/bits/invoke.h:95
#7 0x00007fcdf2f47b07 in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::_M_invoke<0ul, 1ul> (this=0x55c0e1e952c8) at /usr/include/c++/7/thread:234
#8 0x00007fcdf2f47a6d in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::operator() (this=0x55c0e1e952c8) at /usr/include/c++/7/thread:243
#9 0x00007fcdf2f479e2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> > >::_M_run (this=0x55c0e1e952c0) at /usr/include/c++/7/thread:186
#10 0x00007fcdf0a8a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 18 (Thread 0x7fcdd91e5700 (LWP 1846)):
#0 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf1049282 in base::SleepForNanoseconds (nanoseconds=1000000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fcdf1a5fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055c0dfd859cc in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55c0e1a66bd8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:215
#4 0x000055c0dfd88299 in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055c0dfd87ef9 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055c0dfd88d36 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55c0e1a66bd8) at /usr/include/c++/7/thread:234
#7 0x000055c0dfd88ce3 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55c0e1a66bd8) at /usr/include/c++/7/thread:243
#8 0x000055c0dfd88ca2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55c0e1a66bd0) at /usr/include/c++/7/thread:186
#9 0x00007fcdf0a8a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 17 (Thread 0x7fcdd99e6700 (LWP 1845)):
#0 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf1049282 in base::SleepForNanoseconds (nanoseconds=250000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fcdf1a5fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055c0dfd8552c in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55c0e1a66668) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:196
#4 0x000055c0dfd8812f in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055c0dfd87df6 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055c0dfd88d62 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55c0e1a66668) at /usr/include/c++/7/thread:234
#7 0x000055c0dfd88d07 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55c0e1a66668) at /usr/include/c++/7/thread:243
#8 0x000055c0dfd88cc2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55c0e1a66660) at /usr/include/c++/7/thread:186
#9 0x00007fcdf0a8a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7fcdda1e7700 (LWP 1321)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19eb600, timeout=0.099594120165875211) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19eb600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194b560, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194b558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1ae0658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1ae0658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1ae0600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7fcdda9e8700 (LWP 1320)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19eb180, timeout=0.099058294165615735) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19eb180, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194b3a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194b398) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1ae0558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1ae0558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1ae0500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7fcddb1e9700 (LWP 1319)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19ead00, timeout=0.0099812369999199291) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19ead00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194b1e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194b1d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856e58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856e00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7fcddb9ea700 (LWP 1318)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19ea880, timeout=0.099219578165730127) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19ea880, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194b020, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194b018) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856f58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856f58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856f00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7fcddd1ed700 (LWP 1288)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19ea400, timeout=0.098961366165667641) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19ea400, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194ae60, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194ae58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856358) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856300) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7fcddd9ee700 (LWP 1287)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e9f80, timeout=0.099616696165639951) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e9f80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194aca0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194ac98) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856958) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856958) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7fcdde1ef700 (LWP 1286)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e9b00, timeout=0.098930312165748546) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e9b00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194aae0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194aad8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7fcddebf7700 (LWP 1285)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e9680, timeout=0.099417587165817167) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e9680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a920, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a918) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856758) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856700) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7fcddf3f8700 (LWP 405)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e8d80, timeout=0.098981301165622426) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e8d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a220, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a218) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1857458) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1857458) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1857400) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7fcddfbf9700 (LWP 404)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e8900, timeout=0.099926514165645131) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e8900, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a3e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a3d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1857058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1857058) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1857000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7fcde03fa700 (LWP 403)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e8480, timeout=0.09991494016571778) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e8480, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a760, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856858) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7fcde0bfb700 (LWP 402)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e19e8000, timeout=0.09928881616565377) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e19e8000, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a060, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1857258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1857258) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1857200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7fcde13fc700 (LWP 401)):
#0 0x00007fcdf0147a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fcdeb94ade0 in epoll_poll (loop=0x55c0e18ab600, timeout=0.09966540416553471) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fcdeb94e49e in ev_run (loop=0x55c0e18ab600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fcdf1ab8ba3 in ev::loop_ref::run (this=0x55c0e194a5a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fcdee2cd31d in kudu::rpc::ReactorThread::RunThread (this=0x55c0e194a598) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fcdee2c9c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55c0e1856c58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fcdee2d1934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1856c58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1856c00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7fcde1bfd700 (LWP 400)):
#0 0x00007fcdf1ec2fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf19817eb in kudu::ConditionVariable::WaitUntil (this=0x55c0e183f2d8, until=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:87
#2 0x00007fcdf2a59fde in kudu::CountDownLatch::WaitUntil (this=0x55c0e183f2a0, when=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:87
#3 0x00007fcdf2a5a088 in kudu::CountDownLatch::WaitFor (this=0x55c0e183f2a0, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:97
#4 0x00007fcdf1a19166 in kudu::KernelStackWatchdog::RunThread (this=0x55c0e183f200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007fcdf1a18375 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x55c0e1857158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007fcdf1a19aca in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fcdf2d4c856 in std::function<void ()>::operator()() const (this=0x55c0e1857158) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fcdf1abf807 in kudu::Thread::SuperviseThread (arg=0x55c0e1857100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7fcde37e8700 (LWP 399)):
#0 0x00007fcdf1ec732a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf1ab67b5 in kudu::Subprocess::DoWait (this=0x7fcde37e7380, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:817
#2 0x00007fcdf1ab3be8 in kudu::Subprocess::Wait (this=0x7fcde37e7380, wait_status=0x0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:544
#3 0x00007fcdf1a06b7c in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:258
#4 0x00007fcdf1a066ab in kudu::PstackWatcher::RunGdbStackDump (pid=396, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:218
#5 0x00007fcdf1a05f09 in kudu::PstackWatcher::DumpPidStacks (pid=396, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:169
#6 0x00007fcdf1a05e82 in kudu::PstackWatcher::DumpStacks (flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:162
#7 0x00007fcdf2df7087 in kudu::<lambda()>::operator()(void) const (__closure=0x55c0e18bd9f8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:60
#8 0x00007fcdf2df7888 in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#9 0x00007fcdf2df75dc in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#10 0x00007fcdf2df7e6e in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55c0e18bd9f8) at /usr/include/c++/7/thread:234
#11 0x00007fcdf2df7e13 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x55c0e18bd9f8) at /usr/include/c++/7/thread:243
#12 0x00007fcdf2df7dce in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x55c0e18bd9f0) at /usr/include/c++/7/thread:186
#13 0x00007fcdf0a8a6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007fcdf1ebc6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007fcdf014771f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7fcde580d380 (LWP 396)):
#0 0x00007fcdf1ec6d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fcdf1049282 in base::SleepForNanoseconds (nanoseconds=1000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fcdf1a5fa69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055c0dfd868e1 in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody (this=0x55c0e18aba80) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:229
#4 0x00007fcdf1c620ed in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7fcdf1c6382b "the test body", method=<optimized out>, object=0x55c0e18aba80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#5 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x55c0e18aba80, method=<optimized out>, location=location@entry=0x7fcdf1c6382b "the test body") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#6 0x00007fcdf1c56bdb in testing::Test::Run (this=0x55c0e18aba80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#7 0x00007fcdf1c56d9d in testing::TestInfo::Run (this=0x55c0e183e240) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#8 0x00007fcdf1c57377 in testing::TestSuite::Run (this=0x55c0e183e480) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#9 0x00007fcdf1c5777c in testing::internal::UnitTestImpl::RunAllTests (this=<optimized out>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870
#10 0x00007fcdf1c6260d in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7fcdf1c653d8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x55c0e18f6280) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#11 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x55c0e18f6280, method=<optimized out>, location=location@entry=0x7fcdf1c653d8 "auxiliary test code (environments or event listeners)") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#12 0x00007fcdf1c56e63 in testing::UnitTest::Run (this=0x7fcdf1e761e0 <testing::UnitTest::GetInstance()::instance>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444
#13 0x00007fcdf2df81de in RUN_ALL_TESTS () at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#14 0x00007fcdf2df753d in main (argc=1, argv=0x7ffc7a8261e8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:109
#15 0x00007fcdf0047bf7 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#16 0x000055c0dfd84e4a in _start ()
W20240503 14:13:42.116700 1319 meta_cache.cc:302] tablet a9e628012ef24e0ca2b92831feab474e: replica 7a14f28360a64e8895fa011fae4f0781 (127.0.99.4:44245) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:44245: connect: Connection refused (error 111)
************************* END STACKS ***************************
F20240503 14:13:42.205466 399 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1714745622 (unix time) try "date -d @1714745622" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e80000018c) received by PID 396 (TID 0x7fcde37e8700) from PID 396; stack trace: ***
@ 0x7fcdf1ec7980 (unknown) at ??:0
@ 0x7fcdf0064fb7 gsignal at ??:0
@ 0x7fcdf0066921 abort at ??:0
@ 0x7fcdf0d69dcd google::LogMessage::Fail() at ??:0
@ 0x7fcdf0d6db93 google::LogMessage::SendToLog() at ??:0
@ 0x7fcdf0d697cc google::LogMessage::Flush() at ??:0
@ 0x7fcdf0d6af59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fcdf2df71c4 _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at ??:0
@ 0x7fcdf2df7888 _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at ??:0
@ 0x7fcdf2df75dc _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at ??:0
@ 0x7fcdf2df7e6e _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at ??:0
@ 0x7fcdf2df7e13 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at ??:0
@ 0x7fcdf2df7dce _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at ??:0
@ 0x7fcdf0a8a6df (unknown) at ??:0
@ 0x7fcdf1ebc6db start_thread at ??:0
@ 0x7fcdf014771f clone at ??:0