[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from RaftConsensusStressITest
[ RUN ] RaftConsensusStressITest.RemoveReplaceInCycle
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240502 02:00:10.256254 22525 test_util.cc:268] Using random seed: -158816399
I20240502 02:00:10.261860 22525 ts_itest-base.cc:112] Starting cluster with:
I20240502 02:00:10.261900 22525 ts_itest-base.cc:113] --------------
I20240502 02:00:10.261911 22525 ts_itest-base.cc:114] 6 tablet servers
I20240502 02:00:10.261921 22525 ts_itest-base.cc:115] 3 replicas per TS
I20240502 02:00:10.261931 22525 ts_itest-base.cc:116] --------------
2024-05-02T02:00:10Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-02T02:00:10Z Disabled control of system clock
I20240502 02:00:10.269778 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.21.255.126:37013
--webserver_interface=127.21.255.126
--webserver_port=0
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.21.255.126:37013
--minloglevel=2
--raft_prepare_replacement_before_eviction=true
--max_create_tablets_per_ts=40 with env {}
I20240502 02:00:10.433516 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 22537
I20240502 02:00:10.433615 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240502 02:00:10.471946 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.65:0
--local_ip_for_outbound_sockets=127.21.255.65
--webserver_interface=127.21.255.65
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:10.616192 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 22629
I20240502 02:00:10.616290 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240502 02:00:10.618326 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.66:0
--local_ip_for_outbound_sockets=127.21.255.66
--webserver_interface=127.21.255.66
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:10.783000 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 22761
I20240502 02:00:10.783105 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240502 02:00:10.785040 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.67:0
--local_ip_for_outbound_sockets=127.21.255.67
--webserver_interface=127.21.255.67
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:10.929286 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 22893
I20240502 02:00:10.929368 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240502 02:00:10.931210 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.68:0
--local_ip_for_outbound_sockets=127.21.255.68
--webserver_interface=127.21.255.68
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:11.095697 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 23025
I20240502 02:00:11.095782 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240502 02:00:11.097430 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.69:0
--local_ip_for_outbound_sockets=127.21.255.69
--webserver_interface=127.21.255.69
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:11.261498 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 23159
I20240502 02:00:11.261581 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240502 02:00:11.263427 22525 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
/tmp/dist-test-taskDkjNv6/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.21.255.70:0
--local_ip_for_outbound_sockets=127.21.255.70
--webserver_interface=127.21.255.70
--webserver_port=0
--tserver_master_addrs=127.21.255.126:37013
--builtin_ntp_servers=127.21.255.84:42905
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240502 02:00:11.417776 22525 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskDkjNv6/build/debug/bin/kudu as pid 23293
I20240502 02:00:11.417872 22525 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240502 02:00:11.420084 22525 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 02:00:11.503605 22525 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240502 02:00:11.505035 22525 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 1a4347d5a01d470fa0a6de392f1ab548 to finish bootstrapping
I20240502 02:00:11.509569 22525 ts_itest-base.cc:233] Waiting for 1 tablets on tserver d446afde7e1e4bdb97117338b058947f to finish bootstrapping
I20240502 02:00:11.513269 22525 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 2aae1ae3b9d341fa818024fe88766041 to finish bootstrapping
I20240502 02:00:11.517561 22525 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 7f5d52c6029645a5bbc765e3bdcaf13b to finish bootstrapping
I20240502 02:00:11.521238 22525 ts_itest-base.cc:233] Waiting for 0 tablets on tserver bde1dfda778b448bb297069773dc89fa to finish bootstrapping
I20240502 02:00:11.526012 22525 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 92a59bcb31cb4308a4d86be0aa044bd4 to finish bootstrapping
I20240502 02:00:11.530416 22525 test_util.cc:268] Using random seed: -157542227
F20240502 02:00:20.468088 24043 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.queue_time_us, tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.queue_time_us) (tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.run_cpu_time_us, tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.run_cpu_time_us) (tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.run_wall_time_us, tablet-download-pool-0be1300bc8474f9cb829ba8348544c1d.run_wall_time_us) (tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.queue_time_us, tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.queue_time_us) (tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.run_cpu_time_us, tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.run_cpu_time_us) (tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.run_wall_time_us, tablet-download-pool-17eaa7938ffb4900a2ef4317ad36bdd3.run_wall_time_us) (tablet-download-pool-288c37324e49431cb11e2845d2da32f4.queue_time_us, tablet-download-pool-288c37324e49431cb11e2845d2da32f4.queue_time_us) (tablet-download-pool-288c37324e49431cb11e2845d2da32f4.run_cpu_time_us, tablet-download-pool-288c37324e49431cb11e2845d2da32f4.run_cpu_time_us) (tablet-download-pool-288c37324e49431cb11e2845d2da32f4.run_wall_time_us, tablet-download-pool-288c37324e49431cb11e2845d2da32f4.run_wall_time_us) (tablet-download-pool-4d4f1548281d4dca87913195e3eee586.queue_time_us, tablet-download-pool-4d4f1548281d4dca87913195e3eee586.queue_time_us) (tablet-download-pool-4d4f1548281d4dca87913195e3eee586.run_cpu_time_us, tablet-download-pool-4d4f1548281d4dca87913195e3eee586.run_cpu_time_us) (tablet-download-pool-4d4f1548281d4dca87913195e3eee586.run_wall_time_us, tablet-download-pool-4d4f1548281d4dca87913195e3eee586.run_wall_time_us) (tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.queue_time_us, tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.queue_time_us) (tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.run_cpu_time_us, tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.run_cpu_time_us) (tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.run_wall_time_us, tablet-download-pool-5735f9dffdc34137ae05e0dc69500f16.run_wall_time_us) (tablet-download-pool-5d68351be20449e492c747e61fbaff5b.queue_time_us, tablet-download-pool-5d68351be20449e492c747e61fbaff5b.queue_time_us) (tablet-download-pool-5d68351be20449e492c747e61fbaff5b.run_cpu_time_us, tablet-download-pool-5d68351be20449e492c747e61fbaff5b.run_cpu_time_us) (tablet-download-pool-5d68351be20449e492c747e61fbaff5b.run_wall_time_us, tablet-download-pool-5d68351be20449e492c747e61fbaff5b.run_wall_time_us) (tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.queue_time_us, tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.queue_time_us) (tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.run_cpu_time_us, tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.run_cpu_time_us) (tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.run_wall_time_us, tablet-download-pool-6dcc33b8c51b45a5ad16c48db0104db6.run_wall_time_us) (tablet-download-pool-821ab26e7019437db2e1f24e4f140366.queue_time_us, tablet-download-pool-821ab26e7019437db2e1f24e4f140366.queue_time_us) (tablet-download-pool-821ab26e7019437db2e1f24e4f140366.run_cpu_time_us, tablet-download-pool-821ab26e7019437db2e1f24e4f140366.run_cpu_time_us) (tablet-download-pool-821ab26e7019437db2e1f24e4f140366.run_wall_time_us, tablet-download-pool-821ab26e7019437db2e1f24e4f140366.run_wall_time_us) (tablet-download-pool-843e38b0442b4351b49b1add8ea6e4f1.queue_time_us, tablet-download-pool-843e38b0442b4351b49b1add8ea6e4f1.queue_time_us) (tablet-download-pool-925ebc87743b4209811fa978c07937fe.queue_time_us, tablet-download-pool-925ebc87743b4209811fa978c07937fe.queue_time_us) (tablet-download-pool-925ebc87743b4209811fa978c07937fe.run_cpu_time_us, tablet-download-pool-925ebc87743b4209811fa978c07937fe.run_cpu_time_us) (tablet-download-pool-925ebc87743b4209811fa978c07937fe.run_wall_time_us, tablet-download-pool-925ebc87743b4209811fa978c07937fe.run_wall_time_us) (tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.queue_time_us, tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.queue_time_us) (tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.run_cpu_time_us, tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.run_cpu_time_us) (tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.run_wall_time_us, tablet-download-pool-b82c8162dacb4a55ada7b4e0ec79e003.run_wall_time_us) (tablet-download-pool-c4574d734cdd4124b9152eb04676c889.queue_time_us, tablet-download-pool-c4574d734cdd4124b9152eb04676c889.queue_time_us) (tablet-download-pool-c4574d734cdd4124b9152eb04676c889.run_cpu_time_us, tablet-download-pool-c4574d734cdd4124b9152eb04676c889.run_cpu_time_us) (tablet-download-pool-c4574d734cdd4124b9152eb04676c889.run_wall_time_us, tablet-download-pool-c4574d734cdd4124b9152eb04676c889.run_wall_time_us) (tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.queue_time_us, tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.queue_time_us) (tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.run_cpu_time_us, tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.run_cpu_time_us) (tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.run_wall_time_us, tablet-download-pool-c8b80fcaf84f4153872eb11a568af421.run_wall_time_us) (tablet-download-pool-d66974576af347199fe42610fd863e49.queue_time_us, tablet-download-pool-d66974576af347199fe42610fd863e49.queue_time_us) (tablet-download-pool-d66974576af347199fe42610fd863e49.run_cpu_time_us, tablet-download-pool-d66974576af347199fe42610fd863e49.run_cpu_time_us) (tablet-download-pool-d66974576af347199fe42610fd863e49.run_wall_time_us, tablet-download-pool-d66974576af347199fe42610fd863e49.run_wall_time_us) (tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.queue_time_us, tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.queue_time_us) (tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.run_cpu_time_us, tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.run_cpu_time_us) (tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.run_wall_time_us, tablet-download-pool-eb28b07d3b1545868f7ad2f421a9d823.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fb01687ddcd google::LogMessage::Fail() at ??:0
@ 0x7fb016881b93 google::LogMessage::SendToLog() at ??:0
@ 0x7fb01687d7cc google::LogMessage::Flush() at ??:0
@ 0x7fb01687ef59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fb017ce7044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fb017ccbf98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fb017cc9efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fb01a87c77a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fb01a885a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fb01a90676f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fb01a91a4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fb01a905226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fb01a914c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55a95c1ee75c std::function<>::operator()() at ??:0
@ 0x7fb017ccee06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fb017ccf6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fb017cd103b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55a95c1ee75c std::function<>::operator()() at ??:0
@ 0x7fb017cbe807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fb0184fc6db start_thread at ??:0
@ 0x7fb015a3471f clone at ??:0
W20240502 02:00:21.157567 23461 meta_cache.cc:302] tablet 324455f15af84ec6a40557b077d96518: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | HEALTHY | <none> | 17 | 0
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | HEALTHY | <none> | 5 | 0
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 12 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 7 | 0
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | HEALTHY | <none> | 0 | 0
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+--------------------------------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
local_ip_for_outbound_sockets | 127.21.255.65 | experimental | 127.21.255.65:39493
local_ip_for_outbound_sockets | 127.21.255.66 | experimental | 127.21.255.66:37131
local_ip_for_outbound_sockets | 127.21.255.67 | experimental | 127.21.255.67:44479
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
server_dump_info_format | pb | hidden | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.21.255.65:39493
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.21.255.66:37131
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.21.255.67:44479
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+--------------------------------------------------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
time_source | builtin | 127.21.255.65:39493, 127.21.255.67:44479, 127.21.255.70:46489, and 2 other server(s)
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.65:39493, tserver@127.21.255.70:46489, and 3 other server(s)
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 5 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 5 | Yes
D | [config not available] | | |
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | A* B C D~ | 1 | 63 | Yes
D | [config not available] | | |
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 67 | Yes
B | A B* C D~ | 1 | 67 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 65 | No
C | A B C* D~ | 1 | 65 | No
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 61 | Yes
B | A* B C D~ | 1 | 61 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 52 | No
C | A B C* D~ | 2 | 52 | No
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 63 | Yes
B | A* B C | 3 | 63 | Yes
C | [config not available] | | |
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 62 | Yes
B | A* B C D~ | 1 | 62 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 53 | Yes
C | A B C* D~ | 1 | 53 | No
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 60 | No
C | A B C* D~ | 1 | 60 | No
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 71 | Yes
B | A B* C D~ | 1 | 71 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 68 | No
C | A B C* D~ | 1 | 68 | No
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 67 | No
C | A B C* D~ | 1 | 67 | No
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 67 | No
C | A B C* D~ | 2 | 67 | No
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 66 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 65 | No
C | A B C* D~ | 1 | 65 | No
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 68 | Yes
B | A* B C D~ | 1 | 68 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 65 | Yes
B | A* B C D~ | 1 | 65 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 67 | Yes
C | A B C* D~ | 2 | 67 | No
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 60 | Yes
B | A* B C D~ | 1 | 60 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 2 | 59 | Yes
C | A B C* | 2 | -1 | Yes
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): missing [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D~ | 1 | 56 | Yes
B | A* B C D~ | 1 | 56 | Yes
C | [config not available] | | |
D | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* D~ | 1 | 70 | No
C | A B C* D~ | 1 | 70 | No
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = 1a4347d5a01d470fa0a6de392f1ab548
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C D~ | 2 | 62 | No
C | A B* C | 2 | -1 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNDER_REPLICATED | 40 | 11 | 0 | 29 | 0
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 10
First Quartile | 20
Median | 22
Third Quartile | 30
Maximum | 30
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 133
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
F20240502 02:00:24.491966 24605 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us) (tablet-download-pool-324455f15af84ec6a40557b077d96518.queue_time_us, tablet-download-pool-324455f15af84ec6a40557b077d96518.queue_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us) (tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.queue_time_us, tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.queue_time_us) (tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.run_cpu_time_us, tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.run_cpu_time_us) (tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.run_wall_time_us, tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.run_wall_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.queue_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.queue_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_cpu_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_cpu_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_wall_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_wall_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f7351296dcd google::LogMessage::Fail() at ??:0
@ 0x7f735129ab93 google::LogMessage::SendToLog() at ??:0
@ 0x7f73512967cc google::LogMessage::Flush() at ??:0
@ 0x7f7351297f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f7352700044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f73526e4f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f73526e2efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f735529577a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f735529ea45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f735531f76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f73553334f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f735531e226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f735532dc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55e08c24675c std::function<>::operator()() at ??:0
@ 0x7f73526e7e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f73526e86ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f73526ea03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55e08c24675c std::function<>::operator()() at ??:0
@ 0x7f73526d7807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f7352f156db start_thread at ??:0
@ 0x7f735044d71f clone at ??:0
W20240502 02:00:24.617532 23461 meta_cache.cc:302] tablet c9654371c2574eab805599565c1edb91: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:24.758545 23461 connection.cc:535] client connection to 127.21.255.65:39493 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 02:00:25.631441 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:26.633927 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:27.646852 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:28.655738 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:29.656992 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:30.698680 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:31.717530 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:32.740593 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:33.743441 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:34.745432 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:35.855638 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:36.925832 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:37.926308 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:39.031466 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:40.064255 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:41.124650 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:42.223793 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:43.253011 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:44.336419 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:45.338117 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:46.368340 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:47.420672 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:48.498257 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:49.601101 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:50.725261 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:51.876555 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:52.885633 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:54.014187 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:55.057991 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:56.228101 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:57.243058 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:00:58.272716 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:00:59.312168 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:00.372434 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:01.446301 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:02.538653 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:03.640573 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:04.765743 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:05.905900 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:07.057974 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:08.058614 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:09.116736 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:10.321175 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:11.349810 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:12.459870 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:13.509764 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:14.647847 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:15.667428 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:16.878604 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:17.897253 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:18.965107 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:20.161464 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:21.166401 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:22.179091 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:23.200114 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:24.230019 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:24.584810 23458 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 288c37324e49431cb11e2845d2da32f4 after 1 attempt(s): Failed to write to server: 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): Write RPC to 127.21.255.67:44479 timed out after 59.997s (SENT)
W20240502 02:01:25.230849 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
F20240502 02:01:25.547863 27423 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us) (tablet-download-pool-274d0bf494c940c7acf5cadf64eea7a7.queue_time_us, tablet-download-pool-274d0bf494c940c7acf5cadf64eea7a7.queue_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us) (tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.queue_time_us, tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.queue_time_us) (tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.run_cpu_time_us, tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.run_cpu_time_us) (tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.run_wall_time_us, tablet-download-pool-3b86d0d1c8334d83b0a29d90d0fe92f8.run_wall_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.queue_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.queue_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_cpu_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_cpu_time_us) (tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_wall_time_us, tablet-download-pool-9328453f258d48b3a72818a7528d6aa2.run_wall_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f50befa8dcd google::LogMessage::Fail() at ??:0
@ 0x7f50befacb93 google::LogMessage::SendToLog() at ??:0
@ 0x7f50befa87cc google::LogMessage::Flush() at ??:0
@ 0x7f50befa9f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f50c0412044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f50c03f6f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f50c03f4efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f50c2fa777a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f50c2fb0a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f50c303176f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f50c30454f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f50c3030226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f50c303fc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55675a4ac75c std::function<>::operator()() at ??:0
@ 0x7f50c03f9e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f50c03fa6ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f50c03fc03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55675a4ac75c std::function<>::operator()() at ??:0
@ 0x7f50c03e9807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f50c0c276db start_thread at ??:0
@ 0x7f50be15f71f clone at ??:0
W20240502 02:01:26.241936 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:27.251925 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:28.277515 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:29.283839 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:30.291774 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:31.338763 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:32.397742 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:33.401537 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:34.481122 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:35.488852 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:36.492990 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:37.630589 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:38.695807 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:39.699328 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:40.746642 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:41.829672 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:42.835940 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:43.984464 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:45.172354 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:46.177067 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:47.212599 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:48.281495 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:49.284336 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:50.293134 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:51.304260 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:52.328190 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:53.413632 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:54.469772 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:55.481029 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:56.705817 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:01:57.715145 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:58.743331 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:01:59.798890 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:00.851279 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:01.935746 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:03.033578 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:04.144279 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:05.270561 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:06.412894 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:07.431212 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:08.452261 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:09.498378 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:10.540596 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:11.750666 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:12.841706 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:13.917335 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:15.037822 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:16.127614 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:17.149029 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:18.402186 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:19.550189 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:20.551004 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:21.551447 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:22.558224 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:23.579366 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:24.605965 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:24.950976 23461 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet b82c8162dacb4a55ada7b4e0ec79e003 after 343 attempt(s): Failed to write to server: (no server available): Write(tablet: b82c8162dacb4a55ada7b4e0ec79e003, num_ops: 3, num_attempts: 343) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | HEALTHY | <none> | 11 | 0
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 12 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 8 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+---------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.67 | experimental | 127.21.255.67:44479
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.21.255.67:44479
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+---------------------------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.67:44479, 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+--------------------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.70:46489, tserver@127.21.255.67:44479, and 1 other server(s)
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A* B C | 1 | 6 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A* B C | 1 | 64 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 2 | 175 | Yes
C | A B* C D~ | 2 | 175 | Yes
D | [config not available] | | |
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 178 | Yes
C | A B* C | 2 | 178 | Yes
D | [config not available] | | |
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 70 | No
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | A* B C D~ | 3 | 66 | Yes
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 2 | 182 | Yes
C | A B* C D~ | 2 | 182 | Yes
D | [config not available] | | |
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 63 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 2 | 73 | Yes
C | [config not available] | | |
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 180 | Yes
C | A B* C D~ | 1 | 180 | Yes
D | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 2 | 182 | Yes
C | A B* C D~ | 2 | 182 | Yes
D | [config not available] | | |
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 1 | 176 | Yes
C | A B* C D~ | 1 | 176 | Yes
D | [config not available] | | |
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 66 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 61 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | A* B C | 1 | 57 | Yes
C | [config not available] | | |
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
d446afde7e1e4bdb97117338b058947f (127.21.255.66:37131): TS unavailable [NONVOTER]
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
D = d446afde7e1e4bdb97117338b058947f
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | A B* C D~ | 2 | 177 | Yes
C | A B* C D~ | 2 | 177 | Yes
D | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 23 | 0 | 7 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 7
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 131
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:02:25.606005 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
F20240502 02:02:26.456734 29166 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.queue_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_cpu_time_us) (tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us, tablet-download-pool-14cfa3efd53c4ffea10c10081adb000e.run_wall_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.queue_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_cpu_time_us) (tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us, tablet-download-pool-1cf9f352e1684967a7ecbdf3b68b241c.run_wall_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.queue_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_cpu_time_us) (tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us, tablet-download-pool-34bd398299494dfe91d7ae681cddf076.run_wall_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.queue_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_cpu_time_us) (tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us, tablet-download-pool-578789e11edd4fcc8056e4c58d55f226.run_wall_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.queue_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_cpu_time_us) (tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us, tablet-download-pool-6b63a98a609744d38e5308c3b0f98c9b.run_wall_time_us) (tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.queue_time_us, tablet-download-pool-78d4b9e3c1de43ad8d82d6ba9feae70d.queue_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.queue_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_cpu_time_us) (tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us, tablet-download-pool-848db917e32848448ca15d6893ca0362.run_wall_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.queue_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_cpu_time_us) (tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us, tablet-download-pool-8f5f00eab8c4450fac6b54d0676af484.run_wall_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.queue_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_cpu_time_us) (tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us, tablet-download-pool-b486ec8eebe645c49ea871ae41725896.run_wall_time_us) (tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.queue_time_us, tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.queue_time_us) (tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.run_cpu_time_us, tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.run_cpu_time_us) (tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.run_wall_time_us, tablet-download-pool-bf0ffd50903a41168af755d4e2d2415a.run_wall_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.queue_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_cpu_time_us) (tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us, tablet-download-pool-cf2462c2c0894a48918f2d668fb9a349.run_wall_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.queue_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_cpu_time_us) (tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us, tablet-download-pool-e49da3c71629466695413b522d252193.run_wall_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.queue_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_cpu_time_us) (tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us, tablet-download-pool-f031525800ce455b9c8be3a280cb99dd.run_wall_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.queue_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_cpu_time_us) (tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us, tablet-download-pool-f106c4805db94e81b054b24023b55529.run_wall_time_us) (tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.queue_time_us, tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.queue_time_us) (tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.run_cpu_time_us, tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.run_cpu_time_us) (tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.run_wall_time_us, tablet-download-pool-f5e0737abba146a18122b26d1cc499c3.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fe1f9226dcd google::LogMessage::Fail() at ??:0
@ 0x7fe1f922ab93 google::LogMessage::SendToLog() at ??:0
@ 0x7fe1f92267cc google::LogMessage::Flush() at ??:0
@ 0x7fe1f9227f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fe1fa690044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fe1fa674f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fe1fa672efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fe1fd22577a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fe1fd22ea45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fe1fd2af76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fe1fd2c34f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fe1fd2ae226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fe1fd2bdc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5587fef1175c std::function<>::operator()() at ??:0
@ 0x7fe1fa677e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fe1fa6786ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fe1fa67a03b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x5587fef1175c std::function<>::operator()() at ??:0
@ 0x7fe1fa667807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fe1faea56db start_thread at ??:0
@ 0x7fe1f83dd71f clone at ??:0
W20240502 02:02:26.651414 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:26.675861 23458 connection.cc:535] client connection to 127.21.255.67:44479 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240502 02:02:27.652901 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:28.677978 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:29.689301 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:30.699627 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:31.707140 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:32.722168 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:33.726123 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:34.777377 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:35.781891 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:36.792762 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:37.845786 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:38.848260 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:39.887871 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:40.914011 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:41.917820 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:42.989810 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:43.989912 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:44.995512 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:46.018326 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:47.023840 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:48.025806 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:49.041910 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:50.047088 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:51.069334 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:52.074986 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:53.105077 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:54.112152 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:55.142761 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:56.153685 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:02:57.229171 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:02:58.235217 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:02:59.271307 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:00.320536 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:01.383102 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:02.460585 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:03.470692 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:04.561481 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:05.568709 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:06.622896 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:07.698006 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:08.744566 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:09.762495 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:10.825021 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:11.864790 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:12.870857 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:13.890976 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:14.920450 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:15.935882 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:16.950672 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:18.012727 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:19.032459 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:20.111752 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:21.128679 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:22.133126 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:23.145773 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:24.162292 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:25.176738 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:25.520385 23461 batcher.cc:441] Timed out: Failed to write batch of 4 ops to tablet 0be1300bc8474f9cb829ba8348544c1d after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 0be1300bc8474f9cb829ba8348544c1d, num_ops: 4, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:26.185096 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:27.190783 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:28.213757 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:29.249996 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:30.275728 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:31.330157 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:32.364284 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:33.384299 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:34.398511 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:35.425256 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:36.426018 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:37.471840 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:38.471940 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:39.488267 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:40.501272 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:41.512261 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:42.522153 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:43.526177 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:44.603310 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:45.658183 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:46.670416 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:47.705628 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:48.767202 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:03:49.857242 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:50.867579 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:51.878587 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:52.925213 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:53.985081 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:55.059489 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:56.161386 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:03:57.172739 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:58.176337 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:03:59.203848 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:00.243305 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:01.306763 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:02.385177 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:03.472836 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:04.583647 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:05.702157 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:06.842442 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:07.998328 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:09.005244 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:10.057724 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:11.088279 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:12.165120 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:13.220435 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:14.320744 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:15.399941 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:16.522769 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:17.551198 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:18.600224 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:19.751345 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:20.768646 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:21.769924 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:22.776976 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:23.792464 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:24.819365 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:25.827484 23461 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 5d68351be20449e492c747e61fbaff5b after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 5d68351be20449e492c747e61fbaff5b, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.69:45045, tserver@127.21.255.70:46489
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:04:26.221419 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:27.225865 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:28.231204 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:29.234305 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:30.235599 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:31.248572 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:32.303253 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:33.338073 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:34.358078 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:35.371291 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:36.385334 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:37.409075 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:38.438727 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:39.444440 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:40.447371 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:41.465685 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:42.484371 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:43.587237 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:44.601686 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:45.702092 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:46.742280 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:47.769096 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:48.776572 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:49.854167 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:50.953760 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:52.074741 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:53.075996 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:54.165756 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:55.169359 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:56.174479 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:04:57.321379 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:04:58.326141 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:04:59.350806 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:00.393749 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:01.447311 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:02.520805 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:03.611411 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:04.716833 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:05.833083 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:06.971874 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:08.129060 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:09.294175 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:10.473330 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:11.669461 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:12.677716 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:13.806192 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:14.826577 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:15.989774 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:17.036442 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:18.218304 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:19.231966 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:20.279438 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:21.490271 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:22.494582 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:23.503084 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:24.523303 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:25.549427 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:26.236579 23461 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 0be1300bc8474f9cb829ba8348544c1d after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 0be1300bc8474f9cb829ba8348544c1d, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:26.553493 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:27.569429 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:28.606091 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:29.630821 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:30.658067 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:31.665066 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:32.707594 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:33.766170 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:34.772464 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:35.836236 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:36.845345 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:37.860960 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:38.941601 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:39.944476 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:40.946044 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:41.957093 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:43.036526 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:44.149266 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:45.154801 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:46.155444 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:47.285955 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:48.320796 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:49.382284 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:50.474148 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:51.584615 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:52.717692 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:53.723109 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:54.827368 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:55.858847 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:05:57.015376 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:05:58.021322 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:05:59.027213 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:00.054364 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:01.099211 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:02.157948 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:03.231382 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:04.327908 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:05.429661 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:06.554293 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:07.690582 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:08.847432 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:09.849578 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:10.910590 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:11.933928 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:13.020037 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:14.070946 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:15.183871 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:16.191502 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:17.208002 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:18.361187 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:19.361819 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:20.386154 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:21.436098 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:22.442548 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:23.443739 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:24.459075 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:25.486951 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:26.387177 23461 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 0be1300bc8474f9cb829ba8348544c1d after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 0be1300bc8474f9cb829ba8348544c1d, num_ops: 2, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.69:45045, tserver@127.21.255.70:46489
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:06:26.804235 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:27.808050 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:28.820115 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:29.820474 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:30.826674 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:31.854971 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:32.858013 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:33.861922 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:34.871493 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:35.872696 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:36.882191 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:37.887554 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:38.900962 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:39.907167 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:41.006743 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:42.031445 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:43.091112 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:44.195518 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:45.198163 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:46.305481 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:47.307590 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:48.319839 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:49.368145 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:50.441820 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:06:51.544521 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:52.667457 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:53.677757 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:54.763465 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:55.807536 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:56.938227 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:57.957096 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:06:58.971143 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:06:59.997645 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:01.007058 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:02.054179 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:03.124887 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:04.207700 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:05.310920 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:06.424985 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:07.561762 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:08.710947 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:09.874327 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:10.885285 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:11.951996 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:12.957636 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:14.081174 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:15.112915 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:16.255388 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:17.312136 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:18.479913 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:19.498641 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:20.546593 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:21.546811 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:22.567344 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:23.574084 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:24.588814 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:25.612203 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:26.613574 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:26.834820 23461 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 288c37324e49431cb11e2845d2da32f4 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 288c37324e49431cb11e2845d2da32f4, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:27.616189 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:28.625315 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:29.625372 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:30.653601 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:31.655519 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:32.671794 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:33.674454 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:34.685542 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:35.691007 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:36.696898 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:37.731335 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:38.746050 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:39.793524 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:40.824415 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:41.847384 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:42.893074 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:43.900229 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:44.922427 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:45.930735 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:46.970460 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:47.986752 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:49.028177 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:50.084558 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:51.090183 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:52.164388 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:53.193842 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:54.219478 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:55.296901 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:07:56.360303 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:07:57.379261 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:58.390074 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:07:59.400115 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:00.409567 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:01.449532 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:02.455793 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:03.528486 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:04.537981 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:05.601449 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:06.619788 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:07.626443 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:08.631460 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:09.643805 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:10.657506 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:11.700836 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:12.704587 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:13.778813 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:14.845073 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:15.853904 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:16.889055 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:17.999586 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:19.000414 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:20.004328 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:21.087224 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:22.117514 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:23.117672 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:24.130911 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:25.148957 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:26.173473 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:27.000754 23461 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 5d68351be20449e492c747e61fbaff5b after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 5d68351be20449e492c747e61fbaff5b, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.69:45045, tserver@127.21.255.70:46489
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:08:27.446755 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:28.446864 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:29.463057 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:30.472643 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:31.474813 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:32.494349 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:33.548907 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:34.590315 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:35.604806 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:36.632632 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:37.636039 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:38.659395 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:39.704141 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:40.724426 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:41.726276 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:42.748306 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:43.772784 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:44.839084 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:45.846172 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:46.856278 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:47.859253 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:48.880637 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:49.888115 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:50.910718 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:51.974871 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:52.996582 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:54.014171 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:55.032263 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:56.136062 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:08:57.136503 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:08:58.174583 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:08:59.217087 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:00.234679 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:01.269905 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:02.273479 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:03.327173 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:04.331352 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:05.393951 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:06.394915 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:07.471248 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:08.488642 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:09.515503 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:10.518353 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:11.530980 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:12.595278 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:13.628567 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:14.728526 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:15.780159 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:16.910997 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:17.978857 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:18.985638 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:20.111284 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:21.220160 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:22.249945 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:23.401929 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:24.413301 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:25.431913 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:26.460381 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:27.490185 23461 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 821ab26e7019437db2e1f24e4f140366 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 821ab26e7019437db2e1f24e4f140366, num_ops: 1, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:27.673997 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:28.674660 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:29.684501 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:30.693825 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:31.708875 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:32.716389 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:33.718232 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:34.722501 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:35.778079 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:36.828980 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:37.840556 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:38.841948 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:39.893489 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:40.925966 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:41.946825 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:42.953529 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:43.954893 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:44.975946 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:45.991079 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:46.991222 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:47.992771 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:48.998986 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:50.030781 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:51.102813 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:52.200891 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:53.216511 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:09:54.237309 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:55.257700 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:56.350095 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:57.366402 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:09:58.373391 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:09:59.406744 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:00.424948 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:01.454583 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:02.459113 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:03.461975 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:04.529829 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:05.624044 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:06.732986 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:07.857353 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:08.999333 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:10.155946 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:11.168496 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:12.219650 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:13.220036 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:14.337379 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:15.359628 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:16.497995 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:17.547884 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:18.706050 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:19.707242 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:20.761384 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:21.773981 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:22.951186 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:23.955780 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:24.965183 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:25.982218 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:27.012341 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:27.703173 23461 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet d66974576af347199fe42610fd863e49 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: d66974576af347199fe42610fd863e49, num_ops: 3, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.69:45045, tserver@127.21.255.70:46489
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:10:28.112555 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:29.119168 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:30.136819 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:31.137931 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:32.146150 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:33.168864 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:34.220619 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:35.257740 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:36.261184 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:37.297827 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:38.325369 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:39.351208 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:40.358561 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:41.376489 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:42.379184 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:43.403241 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:44.411293 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:45.495227 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:46.517235 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:47.599514 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:48.615304 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:49.624269 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:50.674121 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:51.750350 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:52.848511 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:53.849009 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:10:54.891183 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:55.929253 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:57.013417 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:10:58.016072 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:10:59.036023 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:00.043382 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:01.049827 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:02.055473 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:03.073990 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:04.142707 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:05.223987 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:06.321236 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:07.437264 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:08.563866 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:09.714704 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:10.715284 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:11.765285 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:12.779614 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:13.859054 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:14.899530 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:16.009601 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:17.010217 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:18.013913 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:19.162798 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:20.167346 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:21.219475 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:22.401607 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:23.419978 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:24.420353 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:25.432775 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:26.453387 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:27.455150 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:28.142481 23461 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 288c37324e49431cb11e2845d2da32f4 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 288c37324e49431cb11e2845d2da32f4, num_ops: 2, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:28.461998 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:29.474591 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:30.476526 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:31.477015 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:32.500981 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:33.528854 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:34.529134 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:35.531980 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:36.562719 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:37.577440 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:38.602443 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:39.632740 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:40.662683 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:41.674571 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:42.676088 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:43.706557 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:44.708343 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:45.731606 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:46.757477 23458 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:47.789302 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:48.791424 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:49.815901 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:50.870683 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:51.873095 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:52.889670 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:53.942698 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:54.966331 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:11:56.024947 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:57.039817 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:11:58.052691 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:11:59.081264 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:00.166041 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:01.182843 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:02.214917 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:03.267561 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:04.336371 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:05.423002 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:06.524401 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:07.640146 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:08.641894 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:09.642015 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:10.643043 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:11.664871 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:12.699309 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:13.746376 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:14.808619 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:15.881651 23458 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:16.887071 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:17.904583 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:19.015918 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:20.055308 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:21.102424 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:22.103029 23461 meta_cache.cc:302] tablet f5e0737abba146a18122b26d1cc499c3: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:23.236850 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:24.237768 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:25.242055 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:26.263513 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:27.289251 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:28.322683 23461 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 5735f9dffdc34137ae05e0dc69500f16 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 5735f9dffdc34137ae05e0dc69500f16, num_ops: 3, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.69:45045, tserver@127.21.255.70:46489
Tablet Summary
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:12:28.744650 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:29.749526 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:30.767514 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:31.767555 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:32.776575 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:33.790603 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:34.805815 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:35.808007 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:36.817219 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:37.819171 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:38.829953 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:39.850857 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:40.868626 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:41.927369 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:42.970585 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:43.998189 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:45.000046 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:46.030428 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:47.052631 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:48.083973 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:49.089828 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:50.097081 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:12:51.141094 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:52.149281 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:53.160084 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:54.194080 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:55.195473 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:56.211884 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:57.244601 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:12:58.259476 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:12:59.273109 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:00.336228 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:01.344338 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:02.373732 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:03.422127 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:04.485601 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:05.563334 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:06.575281 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:07.600160 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:08.611155 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:09.624181 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:10.633040 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:11.637629 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:12.669795 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:13.680087 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:14.772193 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:15.808125 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:16.819130 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:17.852960 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:18.859078 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:19.880980 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:20.881534 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:21.909950 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:22.999233 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:24.002447 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:25.050724 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:26.060513 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:27.087301 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:28.117300 23458 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:28.747145 23461 batcher.cc:441] Timed out: Failed to write batch of 4 ops to tablet 0be1300bc8474f9cb829ba8348544c1d after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 0be1300bc8474f9cb829ba8348544c1d, num_ops: 4, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:29.117779 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:30.121884 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:31.129184 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:32.132221 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:33.160104 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:34.171490 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:35.195468 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:36.239912 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:37.250511 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:38.296372 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:39.301276 23458 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:40.317528 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:41.374975 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:42.378321 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:43.388810 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:44.400045 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:45.435026 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:46.441460 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:47.487000 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:48.506788 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:49.509064 23461 meta_cache.cc:302] tablet 6dcc33b8c51b45a5ad16c48db0104db6: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:50.534428 23461 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:51.582038 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:13:52.611297 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:53.708431 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:54.712646 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:55.757234 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:56.803560 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:57.880297 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:13:58.887930 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:13:59.949297 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:00.949738 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:01.971043 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:03.003973 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:04.060170 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:05.129828 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:06.131857 23458 meta_cache.cc:302] tablet 0be1300bc8474f9cb829ba8348544c1d: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:07.196283 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:08.309636 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:09.439414 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:10.450428 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:11.455858 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:12.460021 23461 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:13.514686 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:14.537956 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:15.630178 23458 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:16.674876 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:17.790221 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:18.799363 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:19.816318 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:20.816550 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:21.939019 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:22.962141 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:23.983705 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:24.986693 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:25.999469 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:27.015075 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:28.038509 23461 meta_cache.cc:302] tablet d66974576af347199fe42610fd863e49: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:28.996582 23461 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 821ab26e7019437db2e1f24e4f140366 after 344 attempt(s): Failed to write to server: (no server available): Write(tablet: 821ab26e7019437db2e1f24e4f140366, num_ops: 2, num_attempts: 344) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+----------------------+---------
6f846129f2dc4dc6bce1436dd990fa2a | 127.21.255.126:37013 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+---------------------+-------------------------
builtin_ntp_servers | 127.21.255.84:42905 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+---------------------+-------------+----------+----------------+-----------------
92a59bcb31cb4308a4d86be0aa044bd4 | 127.21.255.70:46489 | HEALTHY | <none> | 19 | 0
bde1dfda778b448bb297069773dc89fa | 127.21.255.69:45045 | HEALTHY | <none> | 11 | 0
1a4347d5a01d470fa0a6de392f1ab548 | 127.21.255.65:39493 | UNAVAILABLE | <none> | n/a | n/a
2aae1ae3b9d341fa818024fe88766041 | 127.21.255.67:44479 | UNAVAILABLE | <none> | n/a | n/a
7f5d52c6029645a5bbc765e3bdcaf13b | 127.21.255.68:45621 | UNAVAILABLE | <none> | n/a | n/a
d446afde7e1e4bdb97117338b058947f | 127.21.255.66:37131 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.21.255.65:39493: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.67:44479: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.68:45621: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.21.255.66:37131: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.21.255.66:37131: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------
ipki_server_key_size | 768 | experimental | 127.21.255.70:46489, 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.69 | experimental | 127.21.255.69:45045
local_ip_for_outbound_sockets | 127.21.255.70 | experimental | 127.21.255.70:46489
never_fsync | true | unsafe,advanced | 127.21.255.70:46489, 127.21.255.69:45045
openssl_security_level_override | 0 | unsafe,hidden | 127.21.255.70:46489, 127.21.255.69:45045
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.21.255.70:46489, 127.21.255.69:45045
rpc_server_allow_ephemeral_ports | true | unsafe | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_format | pb | hidden | 127.21.255.70:46489, 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.21.255.69:45045
server_dump_info_path | /tmp/dist-test-taskDkjNv6/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714615210246688-22525-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.21.255.70:46489
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.21.255.70:46489, 127.21.255.69:45045
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+---------------------+------------------------------------------
builtin_ntp_servers | 127.21.255.84:42905 | 127.21.255.70:46489, 127.21.255.69:45045
time_source | builtin | 127.21.255.70:46489, 127.21.255.69:45045
Version Summary
Version | Servers
-----------------+---------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.21.255.126:37013, tserver@127.21.255.70:46489, tserver@127.21.255.69:45045
Tablet Summary
Tablet 9328453f258d48b3a72818a7528d6aa2 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | 75 | Yes
C | A B C* | 2 | 75 | Yes
Tablet f5e0737abba146a18122b26d1cc499c3 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 90de753fcb02463f9c4e374ca1a4e1b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 123 | Yes
C | A B C* | 4 | 123 | Yes
Tablet c9654371c2574eab805599565c1edb91 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 236 | Yes
C | A B* C | 2 | 236 | Yes
Tablet 88f6eeb0064c46eaa16dfd6188a87922 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 4 | 125 | Yes
C | A B* C | 4 | 125 | Yes
Tablet 78d4b9e3c1de43ad8d82d6ba9feae70d of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet 288c37324e49431cb11e2845d2da32f4 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet f031525800ce455b9c8be3a280cb99dd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet 5d68351be20449e492c747e61fbaff5b of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet e49da3c71629466695413b522d252193 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 61 | Yes
B | A B* C | 2 | 61 | Yes
C | [config not available] | | |
Tablet 5735f9dffdc34137ae05e0dc69500f16 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [NONVOTER]
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
D = bde1dfda778b448bb297069773dc89fa
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
D | A B C D~ | 3 | 66 | Yes
Tablet 3b86d0d1c8334d83b0a29d90d0fe92f8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 242 | Yes
C | A B* C | 2 | 242 | Yes
Tablet 6dcc33b8c51b45a5ad16c48db0104db6 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 34bd398299494dfe91d7ae681cddf076 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 62 | Yes
B | A B* C | 1 | 62 | Yes
C | [config not available] | | |
Tablet 17eaa7938ffb4900a2ef4317ad36bdd3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 124 | Yes
C | A B C* | 4 | 124 | Yes
Tablet 6b63a98a609744d38e5308c3b0f98c9b of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 69 | Yes
B | A B* C | 1 | 69 | Yes
C | [config not available] | | |
Tablet 925ebc87743b4209811fa978c07937fe of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet b486ec8eebe645c49ea871ae41725896 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 78 | Yes
B | A B* C | 1 | 78 | Yes
C | [config not available] | | |
Tablet eb28b07d3b1545868f7ad2f421a9d823 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 126 | Yes
C | A B* C | 3 | 126 | Yes
Tablet 79f3fb5080a04b10a8442a04db21d49c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 240 | Yes
C | A B* C | 1 | 240 | Yes
Tablet 843e38b0442b4351b49b1add8ea6e4f1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 120 | Yes
C | A B C* | 4 | 120 | Yes
Tablet 14cfa3efd53c4ffea10c10081adb000e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 77 | Yes
B | A B* C | 1 | 77 | Yes
C | [config not available] | | |
Tablet c8b80fcaf84f4153872eb11a568af421 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 125 | Yes
C | A B C* | 3 | 125 | Yes
Tablet 8f5f00eab8c4450fac6b54d0676af484 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 4d4f1548281d4dca87913195e3eee586 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | 123 | Yes
C | A B C* | 3 | 123 | Yes
Tablet f106c4805db94e81b054b24023b55529 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet d4c501452e564d46a425d8b8aa24afcd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 5 | 120 | Yes
C | A B C* | 5 | 120 | Yes
Tablet 578789e11edd4fcc8056e4c58d55f226 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 75 | Yes
B | A B* C | 1 | 75 | Yes
C | [config not available] | | |
Tablet b82c8162dacb4a55ada7b4e0ec79e003 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 324455f15af84ec6a40557b077d96518 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 243 | Yes
C | A B* C | 2 | 243 | Yes
Tablet 800f8dadc38c493c8c14f1d81a1a4989 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 4 | 131 | Yes
C | A B C* | 4 | 131 | Yes
Tablet 1c77bab6d7fa45bab2b348d07946f960 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | 234 | Yes
C | A B* C | 1 | 234 | Yes
Tablet 0be1300bc8474f9cb829ba8348544c1d of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 848db917e32848448ca15d6893ca0362 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 76 | Yes
B | A B* C | 2 | 76 | Yes
C | [config not available] | | |
Tablet 821ab26e7019437db2e1f24e4f140366 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet cf2462c2c0894a48918f2d668fb9a349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 2 | 69 | Yes
B | A B* C | 2 | 69 | Yes
C | [config not available] | | |
Tablet d66974576af347199fe42610fd863e49 of table 'RemoveReplaceInCycle' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 2aae1ae3b9d341fa818024fe88766041
C = 7f5d52c6029645a5bbc765e3bdcaf13b
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
Tablet 1cf9f352e1684967a7ecbdf3b68b241c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING [LEADER]
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = bde1dfda778b448bb297069773dc89fa
B = 92a59bcb31cb4308a4d86be0aa044bd4
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 80 | Yes
B | A B* C | 1 | 80 | Yes
C | [config not available] | | |
Tablet c4574d734cdd4124b9152eb04676c889 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 2aae1ae3b9d341fa818024fe88766041
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 3 | 123 | Yes
C | A B* C | 3 | 123 | Yes
Tablet 274d0bf494c940c7acf5cadf64eea7a7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
bde1dfda778b448bb297069773dc89fa (127.21.255.69:45045): RUNNING [LEADER]
92a59bcb31cb4308a4d86be0aa044bd4 (127.21.255.70:46489): RUNNING
All reported replicas are:
A = 7f5d52c6029645a5bbc765e3bdcaf13b
B = bde1dfda778b448bb297069773dc89fa
C = 92a59bcb31cb4308a4d86be0aa044bd4
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | 238 | Yes
C | A B* C | 2 | 238 | Yes
Tablet bf0ffd50903a41168af755d4e2d2415a of table 'TestTable' is unavailable: 3 replica(s) not RUNNING
1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493): TS unavailable [LEADER]
7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621): TS unavailable
2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479): TS unavailable
All reported replicas are:
A = 1a4347d5a01d470fa0a6de392f1ab548
B = 7f5d52c6029645a5bbc765e3bdcaf13b
C = 2aae1ae3b9d341fa818024fe88766041
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | [config not available] | | |
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 0 | 0 | 30 | 10
TestTable | 3 | UNAVAILABLE | 1 | 0 | 0 | 0 | 1
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 0
First Quartile | 11
Median | 30
Third Quartile | 31
Maximum | 34
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 124
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 4 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 4 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 4 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
W20240502 02:14:29.412796 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:30.416646 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:31.419898 23461 meta_cache.cc:302] tablet 821ab26e7019437db2e1f24e4f140366: replica 7f5d52c6029645a5bbc765e3bdcaf13b (127.21.255.68:45621) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.68:45621: connect: Connection refused (error 111)
W20240502 02:14:32.428205 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:33.436326 23458 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:34.445468 23461 meta_cache.cc:302] tablet 5d68351be20449e492c747e61fbaff5b: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:35.462025 23458 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:36.466390 23461 meta_cache.cc:302] tablet 5735f9dffdc34137ae05e0dc69500f16: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:37.466874 23461 meta_cache.cc:302] tablet b82c8162dacb4a55ada7b4e0ec79e003: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
W20240502 02:14:38.481380 23458 meta_cache.cc:302] tablet 925ebc87743b4209811fa978c07937fe: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:39.492995 23458 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 2aae1ae3b9d341fa818024fe88766041 (127.21.255.67:44479) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.67:44479: connect: Connection refused (error 111)
W20240502 02:14:40.503754 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
************************ BEGIN STACKS **************************
[New LWP 22528]
[New LWP 22529]
[New LWP 22530]
[New LWP 22531]
[New LWP 22532]
[New LWP 22533]
[New LWP 22534]
[New LWP 23427]
[New LWP 23428]
[New LWP 23429]
[New LWP 23430]
[New LWP 23458]
[New LWP 23459]
[New LWP 23460]
[New LWP 23461]
[New LWP 23936]
[New LWP 23937]
[New LWP 1513]
[New LWP 1624]
[New LWP 1656]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7fe99edbc380 (LWP 22525) "raft_consensus_" 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7fe99cd97700 (LWP 22528) "raft_consensus_" 0x00007fe9ab47632a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7fe99b3ba700 (LWP 22529) "kernel-watcher-" 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7fe99abb9700 (LWP 22530) "rpc reactor-225" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7fe99a3b8700 (LWP 22531) "rpc reactor-225" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7fe999bb7700 (LWP 22532) "rpc reactor-225" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7fe9993b6700 (LWP 22533) "rpc reactor-225" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7fe998bb5700 (LWP 22534) "rpc reactor-225" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7fe9983b4700 (LWP 23427) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7fe9979ac700 (LWP 23428) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7fe9971ab700 (LWP 23429) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7fe9969aa700 (LWP 23430) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7fe9951a7700 (LWP 23458) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7fe9949a6700 (LWP 23459) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7fe9941a5700 (LWP 23460) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7fe9939a4700 (LWP 23461) "rpc reactor-234" 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7fe9931a3700 (LWP 23936) "raft_consensus_" 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
18 Thread 0x7fe9961a9700 (LWP 23937) "raft_consensus_" 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
19 Thread 0x7fe9919a0700 (LWP 1513) "raft_consensus_" 0x00007fe9ab471ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
20 Thread 0x7fe9921a1700 (LWP 1624) "client-negotiat" 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
21 Thread 0x7fe9929a2700 (LWP 1656) "client-negotiat" 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 21 (Thread 0x7fe9929a2700 (LWP 1656)):
#0 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aaf309d4 in kudu::ConditionVariable::WaitFor (this=0x7fe9929a1690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fe9ab07e8d1 in kudu::ThreadPool::DispatchThread (this=0x555fda65ef00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fe9ab07f6ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x555fdab4f258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fe9ab08103b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fdab4f258) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fdab4f200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 20 (Thread 0x7fe9921a1700 (LWP 1624)):
#0 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aaf309d4 in kudu::ConditionVariable::WaitFor (this=0x7fe9921a0690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007fe9ab07e8d1 in kudu::ThreadPool::DispatchThread (this=0x555fda7acf00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007fe9ab07f6ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x555fda807a58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007fe9ab08103b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda807a58) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda807a00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 19 (Thread 0x7fe9919a0700 (LWP 1513)):
#0 0x00007fe9ab471ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aaf30665 in kudu::ConditionVariable::Wait (this=0x555fdaa89710) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:55
#2 0x00007fe9abcba4c1 in kudu::client::KuduSession::Data::Flush (this=0x555fdaa89680) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/session-internal.cc:257
#3 0x00007fe9abbc4638 in kudu::client::KuduSession::Flush (this=0x555fda59cca0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/client.cc:1524
#4 0x00007fe9ac4eb7e3 in kudu::TestWorkload::WriteThread (this=0x7ffca6b1e750) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/test_workload.cc:204
#5 0x00007fe9ac4f5b8f in std::__invoke_impl<void, void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__f=@0x555fda896850: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7fe9ac4ead88 <kudu::TestWorkload::WriteThread()>, __t=@0x555fda896848: 0x7ffca6b1e750) at /usr/include/c++/7/bits/invoke.h:73
#6 0x00007fe9ac4f5436 in std::__invoke<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__fn=@0x555fda896850: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7fe9ac4ead88 <kudu::TestWorkload::WriteThread()>) at /usr/include/c++/7/bits/invoke.h:95
#7 0x00007fe9ac4f6b07 in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::_M_invoke<0ul, 1ul> (this=0x555fda896848) at /usr/include/c++/7/thread:234
#8 0x00007fe9ac4f6a6d in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::operator() (this=0x555fda896848) at /usr/include/c++/7/thread:243
#9 0x00007fe9ac4f69e2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> > >::_M_run (this=0x555fda896840) at /usr/include/c++/7/thread:186
#10 0x00007fe9aa0396df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 18 (Thread 0x7fe9961a9700 (LWP 23937)):
#0 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aa5f8282 in base::SleepForNanoseconds (nanoseconds=1000000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fe9ab00ea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x0000555fd83679cc in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x555fda7cf5c8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:215
#4 0x0000555fd836a299 in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x0000555fd8369ef9 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x0000555fd836ad36 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x555fda7cf5c8) at /usr/include/c++/7/thread:234
#7 0x0000555fd836ace3 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x555fda7cf5c8) at /usr/include/c++/7/thread:243
#8 0x0000555fd836aca2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x555fda7cf5c0) at /usr/include/c++/7/thread:186
#9 0x00007fe9aa0396df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 17 (Thread 0x7fe9931a3700 (LWP 23936)):
#0 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aa5f8282 in base::SleepForNanoseconds (nanoseconds=250000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fe9ab00ea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x0000555fd836752c in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x555fda7cf718) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:196
#4 0x0000555fd836a12f in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x0000555fd8369df6 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x0000555fd836ad62 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x555fda7cf718) at /usr/include/c++/7/thread:234
#7 0x0000555fd836ad07 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x555fda7cf718) at /usr/include/c++/7/thread:243
#8 0x0000555fd836acc2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x555fda7cf710) at /usr/include/c++/7/thread:186
#9 0x00007fe9aa0396df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7fe9939a4700 (LWP 23461)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda752880, timeout=0.033012305999363889) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda752880, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b3560, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b3558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda848658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda848658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda848600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7fe9941a5700 (LWP 23460)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda752400, timeout=0.099319835162532399) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda752400, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b33a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b3398) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda848558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda848558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda848500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7fe9949a6700 (LWP 23459)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda751f80, timeout=0.098986163162408047) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda751f80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b31e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b31d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bee58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bee58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bee00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7fe9951a7700 (LWP 23458)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda7c0d80, timeout=0.09897332016225846) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda7c0d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b3020, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b3018) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bef58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bef58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bef00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7fe9969aa700 (LWP 23430)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda751b00, timeout=0.099338027162957587) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda751b00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2e60, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5be358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5be358) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5be300) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7fe9971ab700 (LWP 23429)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda751680, timeout=0.099512935163147631) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda751680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2ca0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2c98) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5be958) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5be958) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5be900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7fe9979ac700 (LWP 23428)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda7c1680, timeout=0.099371574162432807) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda7c1680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2ae0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2ad8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5be558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5be558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5be500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7fe9983b4700 (LWP 23427)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda7c1200, timeout=0.099351297163593699) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda7c1200, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2920, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2918) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5be758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5be758) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5be700) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7fe998bb5700 (LWP 22534)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda750d80, timeout=0.098939720166526968) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda750d80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2220, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2218) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bf458) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bf458) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bf400) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7fe9993b6700 (LWP 22533)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda750900, timeout=0.099465024168239324) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda750900, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b23e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b23d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bf058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bf058) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bf000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7fe999bb7700 (LWP 22532)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda750480, timeout=0.098893154166944441) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda750480, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2760, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5be858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5be858) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5be800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7fe99a3b8700 (LWP 22531)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda750000, timeout=0.099428437166352523) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda750000, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b2060, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bf258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bf258) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bf200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7fe99abb9700 (LWP 22530)):
#0 0x00007fe9a96f6a47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007fe9a4ef9de0 in epoll_poll (loop=0x555fda613600, timeout=0.099050069167788024) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007fe9a4efd49e in ev_run (loop=0x555fda613600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007fe9ab067ba3 in ev::loop_ref::run (this=0x555fda6b25a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007fe9a787c31d in kudu::rpc::ReactorThread::RunThread (this=0x555fda6b2598) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007fe9a7878c49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x555fda5bec58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007fe9a7880934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bec58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bec00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7fe99b3ba700 (LWP 22529)):
#0 0x00007fe9ab471fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aaf307eb in kudu::ConditionVariable::WaitUntil (this=0x555fda5a72d8, until=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:87
#2 0x00007fe9ac008fde in kudu::CountDownLatch::WaitUntil (this=0x555fda5a72a0, when=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:87
#3 0x00007fe9ac009088 in kudu::CountDownLatch::WaitFor (this=0x555fda5a72a0, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:97
#4 0x00007fe9aafc8166 in kudu::KernelStackWatchdog::RunThread (this=0x555fda5a7200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007fe9aafc7375 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x555fda5bf158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007fe9aafc8aca in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007fe9ac2fb856 in std::function<void ()>::operator()() const (this=0x555fda5bf158) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007fe9ab06e807 in kudu::Thread::SuperviseThread (arg=0x555fda5bf100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7fe99cd97700 (LWP 22528)):
#0 0x00007fe9ab47632a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9ab0657b5 in kudu::Subprocess::DoWait (this=0x7fe99cd96380, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:817
#2 0x00007fe9ab062be8 in kudu::Subprocess::Wait (this=0x7fe99cd96380, wait_status=0x0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:544
#3 0x00007fe9aafb5b7c in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:258
#4 0x00007fe9aafb56ab in kudu::PstackWatcher::RunGdbStackDump (pid=22525, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:218
#5 0x00007fe9aafb4f09 in kudu::PstackWatcher::DumpPidStacks (pid=22525, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:169
#6 0x00007fe9aafb4e82 in kudu::PstackWatcher::DumpStacks (flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:162
#7 0x00007fe9ac3a6087 in kudu::<lambda()>::operator()(void) const (__closure=0x555fda6259f8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:60
#8 0x00007fe9ac3a6888 in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#9 0x00007fe9ac3a65dc in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#10 0x00007fe9ac3a6e6e in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x555fda6259f8) at /usr/include/c++/7/thread:234
#11 0x00007fe9ac3a6e13 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x555fda6259f8) at /usr/include/c++/7/thread:243
#12 0x00007fe9ac3a6dce in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x555fda6259f0) at /usr/include/c++/7/thread:186
#13 0x00007fe9aa0396df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007fe9ab46b6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007fe9a96f671f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7fe99edbc380 (LWP 22525)):
#0 0x00007fe9ab475d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007fe9aa5f8282 in base::SleepForNanoseconds (nanoseconds=1000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007fe9ab00ea69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x0000555fd83688e1 in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody (this=0x555fda613a80) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:229
#4 0x00007fe9ab2110ed in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7fe9ab21282b "the test body", method=<optimized out>, object=0x555fda613a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#5 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x555fda613a80, method=<optimized out>, location=location@entry=0x7fe9ab21282b "the test body") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#6 0x00007fe9ab205bdb in testing::Test::Run (this=0x555fda613a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#7 0x00007fe9ab205d9d in testing::TestInfo::Run (this=0x555fda5a6240) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#8 0x00007fe9ab206377 in testing::TestSuite::Run (this=0x555fda5a6480) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#9 0x00007fe9ab20677c in testing::internal::UnitTestImpl::RunAllTests (this=<optimized out>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870
#10 0x00007fe9ab21160d in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7fe9ab2143d8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x555fda65e280) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#11 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x555fda65e280, method=<optimized out>, location=location@entry=0x7fe9ab2143d8 "auxiliary test code (environments or event listeners)") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#12 0x00007fe9ab205e63 in testing::UnitTest::Run (this=0x7fe9ab4251e0 <testing::UnitTest::GetInstance()::instance>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444
#13 0x00007fe9ac3a71de in RUN_ALL_TESTS () at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#14 0x00007fe9ac3a653d in main (argc=1, argv=0x7ffca6b1edd8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:109
#15 0x00007fe9a95f6bf7 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#16 0x0000555fd8366e4a in _start ()
W20240502 02:15:09.198951 23461 meta_cache.cc:302] tablet 288c37324e49431cb11e2845d2da32f4: replica 1a4347d5a01d470fa0a6de392f1ab548 (127.21.255.65:39493) has failed: Network error: Client connection negotiation failed: client connection to 127.21.255.65:39493: connect: Connection refused (error 111)
************************* END STACKS ***************************
F20240502 02:15:09.280269 22528 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1714616109 (unix time) try "date -d @1714616109" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e8000057fd) received by PID 22525 (TID 0x7fe99cd97700) from PID 22525; stack trace: ***
@ 0x7fe9ab476980 (unknown) at ??:0
@ 0x7fe9a9613fb7 gsignal at ??:0
@ 0x7fe9a9615921 abort at ??:0
@ 0x7fe9aa318dcd google::LogMessage::Fail() at ??:0
@ 0x7fe9aa31cb93 google::LogMessage::SendToLog() at ??:0
@ 0x7fe9aa3187cc google::LogMessage::Flush() at ??:0
@ 0x7fe9aa319f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fe9ac3a61c4 _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at ??:0
@ 0x7fe9ac3a6888 _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at ??:0
@ 0x7fe9ac3a65dc _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at ??:0
@ 0x7fe9ac3a6e6e _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at ??:0
@ 0x7fe9ac3a6e13 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at ??:0
@ 0x7fe9ac3a6dce _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at ??:0
@ 0x7fe9aa0396df (unknown) at ??:0
@ 0x7fe9ab46b6db start_thread at ??:0
@ 0x7fe9a96f671f clone at ??:0