[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from RaftConsensusStressITest
[ RUN ] RaftConsensusStressITest.RemoveReplaceInCycle
WARNING: Logging before InitGoogleLogging() is written to STDERR
I20240501 08:00:33.725014 396 test_util.cc:268] Using random seed: -510838204
I20240501 08:00:33.732049 396 ts_itest-base.cc:112] Starting cluster with:
I20240501 08:00:33.732108 396 ts_itest-base.cc:113] --------------
I20240501 08:00:33.732128 396 ts_itest-base.cc:114] 6 tablet servers
I20240501 08:00:33.732146 396 ts_itest-base.cc:115] 3 replicas per TS
I20240501 08:00:33.732161 396 ts_itest-base.cc:116] --------------
2024-05-01T08:00:33Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
2024-05-01T08:00:33Z Disabled control of system clock
I20240501 08:00:33.742614 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
master
run
--ipki_ca_key_size=768
--tsk_num_rsa_bits=512
--rpc_bind_addresses=127.0.99.62:38485
--webserver_interface=127.0.99.62
--webserver_port=0
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--rpc_reuseport=true
--master_addresses=127.0.99.62:38485
--minloglevel=2
--raft_prepare_replacement_before_eviction=true
--max_create_tablets_per_ts=40 with env {}
I20240501 08:00:33.980332 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 408
I20240501 08:00:33.980491 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/wal/instance
I20240501 08:00:34.135501 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.1:0
--local_ip_for_outbound_sockets=127.0.99.1
--webserver_interface=127.0.99.1
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:34.331883 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 499
I20240501 08:00:34.331990 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/wal/instance
I20240501 08:00:34.334205 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.2:0
--local_ip_for_outbound_sockets=127.0.99.2
--webserver_interface=127.0.99.2
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:34.551014 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 630
I20240501 08:00:34.551115 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/wal/instance
I20240501 08:00:34.553265 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.3:0
--local_ip_for_outbound_sockets=127.0.99.3
--webserver_interface=127.0.99.3
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:34.749436 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 761
I20240501 08:00:34.749562 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/wal/instance
I20240501 08:00:34.752136 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.4:0
--local_ip_for_outbound_sockets=127.0.99.4
--webserver_interface=127.0.99.4
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:34.948467 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 892
I20240501 08:00:34.948572 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/wal/instance
I20240501 08:00:34.950893 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.5:0
--local_ip_for_outbound_sockets=127.0.99.5
--webserver_interface=127.0.99.5
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:35.147230 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 1023
I20240501 08:00:35.147341 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/wal/instance
I20240501 08:00:35.149660 396 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
/tmp/dist-test-taskvJpsOF/build/debug/bin/kudu
--fs_wal_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/wal
--fs_data_dirs=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data
--block_manager=log
--webserver_interface=localhost
--never_fsync
--enable_minidumps=false
--redact=none
--metrics_log_interval_ms=1000
--log_dir=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/logs
--server_dump_info_path=/tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb
--server_dump_info_format=pb
--rpc_server_allow_ephemeral_ports
--unlock_experimental_flags
--unlock_unsafe_flags
--logtostderr
--logbuflevel=-1
--ipki_server_key_size=768
--openssl_security_level_override=0
tserver
run
--rpc_bind_addresses=127.0.99.6:0
--local_ip_for_outbound_sockets=127.0.99.6
--webserver_interface=127.0.99.6
--webserver_port=0
--tserver_master_addrs=127.0.99.62:38485
--builtin_ntp_servers=127.0.99.20:36269
--builtin_ntp_poll_interval_ms=100
--ntp_initial_sync_wait_secs=10
--time_source=builtin
--minloglevel=2
--tablet_copy_early_session_timeout_prob=0.25
--raft_prepare_replacement_before_eviction=true
--follower_unavailable_considered_failed_sec=1
--consensus_rpc_timeout_ms=1000
--heartbeat_interval_ms=250
--raft_heartbeat_interval_ms=100 with env {}
I20240501 08:00:35.346069 396 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskvJpsOF/build/debug/bin/kudu as pid 1154
I20240501 08:00:35.346185 396 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/wal/instance
I20240501 08:00:35.361213 396 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240501 08:00:35.585848 396 external_mini_cluster.cc:930] 6 TS(s) registered with all masters
I20240501 08:00:35.588002 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver a80601f7114d41e2abd9b423d514e048 to finish bootstrapping
I20240501 08:00:35.595446 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 5b14472270734421a5284e2b5e7ee17a to finish bootstrapping
I20240501 08:00:35.601406 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 6ee13bbcbfc142cfac96ca0721cd8081 to finish bootstrapping
I20240501 08:00:35.607760 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 55f6008aeaae4a8d9d2c0dc6405be801 to finish bootstrapping
I20240501 08:00:35.613107 396 ts_itest-base.cc:233] Waiting for 0 tablets on tserver 3b978884440b466f8682dbad60b636d8 to finish bootstrapping
I20240501 08:00:35.619647 396 ts_itest-base.cc:233] Waiting for 1 tablets on tserver 883d17aecdcb49d886fb9b5af1b1b571 to finish bootstrapping
I20240501 08:00:35.625574 396 test_util.cc:268] Using random seed: -508937634
F20240501 08:00:46.288877 1971 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.queue_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.queue_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_cpu_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_cpu_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_wall_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_wall_time_us) (tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.queue_time_us, tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.queue_time_us) (tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.run_cpu_time_us, tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.run_cpu_time_us) (tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.run_wall_time_us, tablet-download-pool-15ef004a6c104b9fa5f712096b4fcbdd.run_wall_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.queue_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.queue_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_cpu_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_cpu_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_wall_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_wall_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.queue_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.queue_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_cpu_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_cpu_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_wall_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_wall_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.queue_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.queue_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_cpu_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_cpu_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_wall_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_wall_time_us) (tablet-download-pool-61e49eb5434f451892cd1ea845ca8af3.queue_time_us, tablet-download-pool-61e49eb5434f451892cd1ea845ca8af3.queue_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.queue_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.queue_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_cpu_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_cpu_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_wall_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_wall_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.queue_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.queue_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_cpu_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_cpu_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_wall_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_wall_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.queue_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.queue_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_cpu_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_cpu_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_wall_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_wall_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.queue_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.queue_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_cpu_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_cpu_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_wall_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_wall_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.queue_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.queue_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_cpu_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_cpu_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_wall_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_wall_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.queue_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.queue_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_cpu_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_cpu_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_wall_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_wall_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.queue_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.queue_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_cpu_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_cpu_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_wall_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_wall_time_us) (tablet-download-pool-c712ac72cb614d089d660062a92942c1.queue_time_us, tablet-download-pool-c712ac72cb614d089d660062a92942c1.queue_time_us) (tablet-download-pool-c712ac72cb614d089d660062a92942c1.run_cpu_time_us, tablet-download-pool-c712ac72cb614d089d660062a92942c1.run_cpu_time_us) (tablet-download-pool-c712ac72cb614d089d660062a92942c1.run_wall_time_us, tablet-download-pool-c712ac72cb614d089d660062a92942c1.run_wall_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.queue_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.queue_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_cpu_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_cpu_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_wall_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7f4524b31dcd google::LogMessage::Fail() at ??:0
@ 0x7f4524b35b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f4524b317cc google::LogMessage::Flush() at ??:0
@ 0x7f4524b32f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f4525f9b044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7f4525f7ff98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7f4525f7defc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7f4528b3077a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7f4528b39a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7f4528bba76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7f4528bce4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7f4528bb9226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7f4528bc8c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x560234b8775c std::function<>::operator()() at ??:0
@ 0x7f4525f82e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7f4525f836ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7f4525f8503b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x560234b8775c std::function<>::operator()() at ??:0
@ 0x7f4525f72807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7f45267b06db start_thread at ??:0
@ 0x7f4523ce871f clone at ??:0
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+---------+----------+----------------+-----------------
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | HEALTHY | <none> | 9 | 0
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | HEALTHY | <none> | 7 | 0
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | HEALTHY | <none> | 9 | 0
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 0 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 4 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 5 | 0
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_server_key_size | 768 | experimental | all 6 server(s) checked
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:46683
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.4 | experimental | 127.0.99.4:43085
local_ip_for_outbound_sockets | 127.0.99.5 | experimental | 127.0.99.5:34987
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | all 6 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 6 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 6 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 6 server(s) checked
server_dump_info_format | pb | hidden | all 6 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:46683
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-3/data/info.pb | hidden | 127.0.99.4:43085
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.0.99.5:34987
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | all 6 server(s) checked
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 6 server(s) checked
time_source | builtin | all 6 server(s) checked
Version Summary
Version | Servers
-----------------+-------------------------
1.18.0-SNAPSHOT | all 7 server(s) checked
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A* B C | 2 | -1 | Yes
Tablet 390ca9e7928f4affb23b9852624d6e7d of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | A* B C D~ | 1 | 59 | Yes
Tablet 15ef004a6c104b9fa5f712096b4fcbdd of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A* B C D~ | 2 | 61 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): not running
State: STOPPING
Data state: TABLET_DATA_READY
Last status: No bootstrap required, opened a new log
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | A B C | 1 | -1 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | A B C | 2 | -1 | Yes
Tablet 1457c812ae224d26aa0bf2970ff0f8be of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 65 | Yes
Tablet bd276a242f11451d85f8d3b3b6a224a9 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A* B C | 2 | -1 | Yes
Tablet d2be42fb0d6143b79ea8e7f4bba4f4eb of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A* B C | 2 | -1 | Yes
Tablet 61e49eb5434f451892cd1ea845ca8af3 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+--------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 52b9ee325c6c488aa15ee522a36b486a of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 65 | Yes
Tablet a72706b2302047b2ae4b15b2a7aad5e7 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | -1 | Yes
B | A B* C | 1 | -1 | Yes
C | A B* C D~ | 1 | 59 | Yes
Tablet aba0191c5a4f4f7fb821b2f6bafc91d0 of table 'RemoveReplaceInCycle' is conflicted: 3 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B C | 2 | -1 | Yes
B | A B C | 2 | -1 | Yes
C | A B* C D~ | 3 | 59 | Yes
Tablet c712ac72cb614d089d660062a92942c1 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | -1 | Yes
B | A* B C | 1 | -1 | Yes
C | A* B C D~ | 1 | 67 | Yes
Tablet 416c2905133b47cd9eb056c890af2e53 of table 'RemoveReplaceInCycle' is conflicted: 1 replicas' active configs disagree with the leader master's
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [LEADER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | A B C* | 1 | -1 | Yes
C | A B* C D~ | 2 | 63 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+--------------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | CONSENSUS_MISMATCH | 40 | 26 | 0 | 1 | 13
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 20
First Quartile | 20
Median | 21
Third Quartile | 21
Maximum | 21
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 123
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
==================
Errors:
==================
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:00:47.061247 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | HEALTHY | <none> | 11 | 0
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | HEALTHY | <none> | 11 | 0
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 0 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 9 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 10 | 0
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-----------------------------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.2 | experimental | 127.0.99.2:46683
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.5 | experimental | 127.0.99.5:34987
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
server_dump_info_format | pb | hidden | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-1/data/info.pb | hidden | 127.0.99.2:46683
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-4/data/info.pb | hidden | 127.0.99.5:34987
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+-----------------------------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
time_source | builtin | 127.0.99.5:34987, 127.0.99.2:46683, 127.0.99.3:37779, and 2 other server(s)
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.5:34987, tserver@127.0.99.3:37779, and 3 other server(s)
Tablet Summary
Tablet bf03bf83410e4599b1f089271e40f966 of table 'TestTable' is under-replicated: 1 replica(s) not RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
All reported replicas are:
A = 883d17aecdcb49d886fb9b5af1b1b571
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | A B C* | 1 | -1 | Yes
B | [config not available] | | |
C | A B C* | 1 | -1 | Yes
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 61 | Yes
B | A* B C | 1 | 61 | Yes
C | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 62 | Yes
B | A* B C | 2 | 62 | Yes
C | [config not available] | | |
Tablet d63cab148cc54f3ba46ee040f8372867 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 66 | Yes
B | A* B C | 1 | 66 | Yes
C | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 3 | -1 | Yes
C | A B C* | 3 | -1 | Yes
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 65 | Yes
B | A* B C | 1 | 65 | Yes
C | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 60 | Yes
B | A B* C | 1 | 60 | Yes
C | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 55 | Yes
B | A B* C | 1 | 55 | Yes
C | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 795ca2f3afe54abbb9d71d9a9bff2ca0 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 3ace2fd687b649cdb1094d9fd09fc045 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 1 | -1 | Yes
C | A B* C | 1 | -1 | Yes
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 67 | Yes
B | A B* C | 1 | 67 | Yes
C | [config not available] | | |
Tablet a49fa3f295ac416b8f161d0270ff7db7 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 1 | -1 | Yes
C | A B C* | 1 | -1 | Yes
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): RUNNING [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 71 | Yes
B | A B* C | 1 | 71 | Yes
C | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet e120c90668c54a1eb2051764cb753f5c of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | A B* C | 2 | -1 | Yes
C | A B* C | 2 | -1 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): RUNNING
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | A B C* | 2 | -1 | Yes
C | A B C* | 2 | -1 | Yes
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+------------------+---------------+---------+------------+------------------+-------------
RemoveReplaceInCycle | 3 | UNDER_REPLICATED | 40 | 12 | 0 | 28 | 0
TestTable | 3 | UNDER_REPLICATED | 1 | 0 | 0 | 1 | 0
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 11
First Quartile | 20
Median | 21
Third Quartile | 22
Maximum | 29
Tablet Replica Count Outliers
Type | UUID | Host | Replica Count
-------+----------------------------------+------------------+---------------
Big | 55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | 29
Small | 6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | 11
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 123
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 1 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 1 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 1 of 6 had errors
Corruption: table consistency check error: 2 out of 2 table(s) are not healthy
FAILED
F20240501 08:00:47.830237 2310 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.queue_time_us, tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.queue_time_us) (tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.run_cpu_time_us, tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.run_cpu_time_us) (tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.run_wall_time_us, tablet-download-pool-1ab58726b4904b95a2c6d2a5daca65b8.run_wall_time_us) (tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.queue_time_us, tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.queue_time_us) (tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.run_cpu_time_us, tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.run_cpu_time_us) (tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.run_wall_time_us, tablet-download-pool-1d2de061358a43cdbd1b96e945dd8153.run_wall_time_us) (tablet-download-pool-2144cdf57aa04828901e481213cddd02.queue_time_us, tablet-download-pool-2144cdf57aa04828901e481213cddd02.queue_time_us) (tablet-download-pool-2144cdf57aa04828901e481213cddd02.run_cpu_time_us, tablet-download-pool-2144cdf57aa04828901e481213cddd02.run_cpu_time_us) (tablet-download-pool-2144cdf57aa04828901e481213cddd02.run_wall_time_us, tablet-download-pool-2144cdf57aa04828901e481213cddd02.run_wall_time_us) (tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.queue_time_us, tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.queue_time_us) (tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.run_cpu_time_us, tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.run_cpu_time_us) (tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.run_wall_time_us, tablet-download-pool-2c9e8ce22edd4d479d92fef37147933e.run_wall_time_us) (tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.queue_time_us, tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.queue_time_us) (tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.run_cpu_time_us, tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.run_cpu_time_us) (tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.run_wall_time_us, tablet-download-pool-3428d0607c4e4abd9ce794ddfa180915.run_wall_time_us) (tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.queue_time_us, tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.queue_time_us) (tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.run_cpu_time_us, tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.run_cpu_time_us) (tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.run_wall_time_us, tablet-download-pool-3ace2fd687b649cdb1094d9fd09fc045.run_wall_time_us) (tablet-download-pool-3d3d4a35a725452da7e1542ce5c4e8b9.queue_time_us, tablet-download-pool-3d3d4a35a725452da7e1542ce5c4e8b9.queue_time_us) (tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.queue_time_us, tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.queue_time_us) (tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.run_cpu_time_us, tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.run_cpu_time_us) (tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.run_wall_time_us, tablet-download-pool-795ca2f3afe54abbb9d71d9a9bff2ca0.run_wall_time_us) (tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.queue_time_us, tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.queue_time_us) (tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.run_cpu_time_us, tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.run_cpu_time_us) (tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.run_wall_time_us, tablet-download-pool-a49fa3f295ac416b8f161d0270ff7db7.run_wall_time_us) (tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.queue_time_us, tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.queue_time_us) (tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.run_cpu_time_us, tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.run_cpu_time_us) (tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.run_wall_time_us, tablet-download-pool-b27ed99ebb484a798006727a3fdba1fa.run_wall_time_us) (tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.queue_time_us, tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.queue_time_us) (tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.run_cpu_time_us, tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.run_cpu_time_us) (tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.run_wall_time_us, tablet-download-pool-c540bda6e2aa4d7dbb16078aa856b114.run_wall_time_us) (tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.queue_time_us, tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.queue_time_us) (tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.run_cpu_time_us, tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.run_cpu_time_us) (tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.run_wall_time_us, tablet-download-pool-d63cab148cc54f3ba46ee040f8372867.run_wall_time_us) (tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.queue_time_us, tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.queue_time_us) (tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.run_cpu_time_us, tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.run_cpu_time_us) (tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.run_wall_time_us, tablet-download-pool-e120c90668c54a1eb2051764cb753f5c.run_wall_time_us) (tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.queue_time_us, tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.queue_time_us) (tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.run_cpu_time_us, tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.run_cpu_time_us) (tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.run_wall_time_us, tablet-download-pool-f8180cedc3f14f9f95855877ba5f7cb0.run_wall_time_us) (tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.queue_time_us, tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.queue_time_us) (tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.run_cpu_time_us, tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.run_cpu_time_us) (tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.run_wall_time_us, tablet-download-pool-fea0a0c497e044e9b383ebaf6f3a675a.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fc5ee421dcd google::LogMessage::Fail() at ??:0
@ 0x7fc5ee425b93 google::LogMessage::SendToLog() at ??:0
@ 0x7fc5ee4217cc google::LogMessage::Flush() at ??:0
@ 0x7fc5ee422f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fc5ef88b044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fc5ef86ff98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fc5ef86defc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fc5f242077a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fc5f2429a45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fc5f24aa76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fc5f24be4f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fc5f24a9226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fc5f24b8c0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55782625575c std::function<>::operator()() at ??:0
@ 0x7fc5ef872e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fc5ef8736ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fc5ef87503b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x55782625575c std::function<>::operator()() at ??:0
@ 0x7fc5ef862807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fc5f00a06db start_thread at ??:0
@ 0x7fc5ed5d871f clone at ??:0
W20240501 08:00:48.739244 1318 connection.cc:535] client connection to 127.0.99.2:46683 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:00:48.739368 1318 meta_cache.cc:302] tablet 61e49eb5434f451892cd1ea845ca8af3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
W20240501 08:00:49.760794 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:00:50.809149 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:51.849363 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:52.860461 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:53.894203 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:00:54.999414 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:56.030223 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:57.030277 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:00:58.032838 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:00:59.067680 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:00.096982 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:01.107597 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:02.124672 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:03.147903 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:04.164400 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:05.237660 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:06.251840 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:07.307013 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:08.483808 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:09.487735 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:10.493985 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:11.545753 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:12.623081 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:13.624346 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:14.625491 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:15.675499 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:16.714298 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:17.805131 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:18.885507 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:20.008358 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:21.009941 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:22.176223 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:23.202147 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:24.254530 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:25.336300 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:26.414848 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:27.516666 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:28.653620 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:29.776412 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:30.782876 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:31.805333 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:32.985248 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:34.031213 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:35.093008 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:36.166121 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:37.246464 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:38.503386 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:39.616937 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:01:40.738395 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:41.871225 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:43.011921 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:44.029912 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:45.030928 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:46.043424 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:47.063799 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
F20240501 08:01:47.072755 5580 trace_metrics.cc:68] Check failed: g_intern_map->size() < 100 (100 vs. 100) Too many interned strings: (MaintenanceMgr.queue_time_us, MaintenanceMgr.queue_time_us) (MaintenanceMgr.run_cpu_time_us, MaintenanceMgr.run_cpu_time_us) (MaintenanceMgr.run_wall_time_us, MaintenanceMgr.run_wall_time_us) (apply.queue_time_us, apply.queue_time_us) (apply.run_cpu_time_us, apply.run_cpu_time_us) (apply.run_wall_time_us, apply.run_wall_time_us) (client-negotiator.queue_time_us, client-negotiator.queue_time_us) (client-negotiator.run_cpu_time_us, client-negotiator.run_cpu_time_us) (client-negotiator.run_wall_time_us, client-negotiator.run_wall_time_us) (compiler_manager_pool.queue_time_us, compiler_manager_pool.queue_time_us) (compiler_manager_pool.run_cpu_time_us, compiler_manager_pool.run_cpu_time_us) (compiler_manager_pool.run_wall_time_us, compiler_manager_pool.run_wall_time_us) (dirs.queue_time_us, dirs.queue_time_us) (dirs.run_cpu_time_us, dirs.run_cpu_time_us) (dirs.run_wall_time_us, dirs.run_wall_time_us) (dns-resolver.queue_time_us, dns-resolver.queue_time_us) (dns-resolver.run_cpu_time_us, dns-resolver.run_cpu_time_us) (dns-resolver.run_wall_time_us, dns-resolver.run_wall_time_us) (log-alloc.queue_time_us, log-alloc.queue_time_us) (log-alloc.run_cpu_time_us, log-alloc.run_cpu_time_us) (log-alloc.run_wall_time_us, log-alloc.run_wall_time_us) (prepare.queue_time_us, prepare.queue_time_us) (prepare.run_cpu_time_us, prepare.run_cpu_time_us) (prepare.run_wall_time_us, prepare.run_wall_time_us) (raft.queue_time_us, raft.queue_time_us) (raft.run_cpu_time_us, raft.run_cpu_time_us) (raft.run_wall_time_us, raft.run_wall_time_us) (repair_pool.queue_time_us, repair_pool.queue_time_us) (repair_pool.run_cpu_time_us, repair_pool.run_cpu_time_us) (repair_pool.run_wall_time_us, repair_pool.run_wall_time_us) (server-negotiator.queue_time_us, server-negotiator.queue_time_us) (server-negotiator.run_cpu_time_us, server-negotiator.run_cpu_time_us) (server-negotiator.run_wall_time_us, server-negotiator.run_wall_time_us) (tablet-copy.queue_time_us, tablet-copy.queue_time_us) (tablet-copy.run_cpu_time_us, tablet-copy.run_cpu_time_us) (tablet-copy.run_wall_time_us, tablet-copy.run_wall_time_us) (tablet-delete.queue_time_us, tablet-delete.queue_time_us) (tablet-delete.run_cpu_time_us, tablet-delete.run_cpu_time_us) (tablet-delete.run_wall_time_us, tablet-delete.run_wall_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.queue_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.queue_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_cpu_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_cpu_time_us) (tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_wall_time_us, tablet-download-pool-1457c812ae224d26aa0bf2970ff0f8be.run_wall_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.queue_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.queue_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_cpu_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_cpu_time_us) (tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_wall_time_us, tablet-download-pool-390ca9e7928f4affb23b9852624d6e7d.run_wall_time_us) (tablet-download-pool-416c2905133b47cd9eb056c890af2e53.queue_time_us, tablet-download-pool-416c2905133b47cd9eb056c890af2e53.queue_time_us) (tablet-download-pool-416c2905133b47cd9eb056c890af2e53.run_cpu_time_us, tablet-download-pool-416c2905133b47cd9eb056c890af2e53.run_cpu_time_us) (tablet-download-pool-416c2905133b47cd9eb056c890af2e53.run_wall_time_us, tablet-download-pool-416c2905133b47cd9eb056c890af2e53.run_wall_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.queue_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.queue_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_cpu_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_cpu_time_us) (tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_wall_time_us, tablet-download-pool-52b9ee325c6c488aa15ee522a36b486a.run_wall_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.queue_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.queue_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_cpu_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_cpu_time_us) (tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_wall_time_us, tablet-download-pool-52f43c86fc584a6b87ddbf2cfa150399.run_wall_time_us) (tablet-download-pool-61e49eb5434f451892cd1ea845ca8af3.queue_time_us, tablet-download-pool-61e49eb5434f451892cd1ea845ca8af3.queue_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.queue_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.queue_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_cpu_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_cpu_time_us) (tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_wall_time_us, tablet-download-pool-7cf873316527452990569bfcc6bfe238.run_wall_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.queue_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.queue_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_cpu_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_cpu_time_us) (tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_wall_time_us, tablet-download-pool-8b471bd5fab04ca2b38fea49facb6a62.run_wall_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.queue_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.queue_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_cpu_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_cpu_time_us) (tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_wall_time_us, tablet-download-pool-a72706b2302047b2ae4b15b2a7aad5e7.run_wall_time_us) (tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.queue_time_us, tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.queue_time_us) (tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.run_cpu_time_us, tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.run_cpu_time_us) (tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.run_wall_time_us, tablet-download-pool-aba0191c5a4f4f7fb821b2f6bafc91d0.run_wall_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.queue_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.queue_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_cpu_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_cpu_time_us) (tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_wall_time_us, tablet-download-pool-ac8116c4d57d4891b118fb098266a104.run_wall_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.queue_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.queue_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_cpu_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_cpu_time_us) (tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_wall_time_us, tablet-download-pool-b1266d9b2ba14491a43470c4c3a672d3.run_wall_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.queue_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.queue_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_cpu_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_cpu_time_us) (tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_wall_time_us, tablet-download-pool-ba20ceb60c1541749c404f5d98d7cd5e.run_wall_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.queue_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.queue_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_cpu_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_cpu_time_us) (tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_wall_time_us, tablet-download-pool-be6a13abc3b143dd97f3edfa0dd50005.run_wall_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.queue_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.queue_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_cpu_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_cpu_time_us) (tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_wall_time_us, tablet-download-pool-ff8fbf4522a44e8887754461665a4349.run_wall_time_us) (tablet-open.queue_time_us, tablet-open.queue_time_us) (tablet-open.run_cpu_time_us, tablet-open.run_cpu_time_us) (tablet-open.run_wall_time_us, tablet-open.run_wall_time_us) (txn-commit.queue_time_us, txn-commit.queue_time_us) (txn-commit.run_cpu_time_us, txn-commit.run_cpu_time_us) (txn-commit.run_wall_time_us, txn-commit.run_wall_time_us) (txn-participant-registration.queue_time_us, txn-participant-registration.queue_time_us) (txn-participant-registration.run_cpu_time_us, txn-participant-registration.run_cpu_time_us) (txn-participant-registration.run_wall_time_us, txn-participant-registration.run_wall_time_us) (txn-status-manager.queue_time_us, txn-status-manager.queue_time_us) (txn-status-manager.run_cpu_time_us, txn-status-manager.run_cpu_time_us) (txn-status-manager.run_wall_time_us, txn-status-manager.run_wall_time_us) (txn-status-tablet-reload.queue_time_us, txn-status-tablet-reload.queue_time_us) (txn-status-tablet-reload.run_cpu_time_us, txn-status-tablet-reload.run_cpu_time_us) (txn-status-tablet-reload.run_wall_time_us, txn-status-tablet-reload.run_wall_time_us) (wal-append.queue_time_us, wal-append.queue_time_us) (wal-append.run_cpu_time_us, wal-append.run_cpu_time_us) (wal-append.run_wall_time_us, wal-append.run_wall_time_us)
*** Check failure stack trace: ***
@ 0x7fc24c3d5dcd google::LogMessage::Fail() at ??:0
@ 0x7fc24c3d9b93 google::LogMessage::SendToLog() at ??:0
@ 0x7fc24c3d57cc google::LogMessage::Flush() at ??:0
@ 0x7fc24c3d6f59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7fc24d83f044 kudu::TraceMetrics::InternName() at ??:0
@ 0x7fc24d823f98 kudu::ThreadPool::ThreadPool() at ??:0
@ 0x7fc24d821efc kudu::ThreadPoolBuilder::Build() at ??:0
@ 0x7fc2503d477a kudu::tserver::TabletCopyClient::TabletCopyClient() at ??:0
@ 0x7fc2503dda45 kudu::tserver::RemoteTabletCopyClient::RemoteTabletCopyClient() at ??:0
@ 0x7fc25045e76f kudu::tserver::TSTabletManager::RunTabletCopy() at ??:0
@ 0x7fc2504724f6 kudu::tserver::TabletCopyRunnable::Run() at ??:0
@ 0x7fc25045d226 _ZZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS_6StatusENS0_24TabletServerErrorPB_CodeEEEENKUlvE_clEv at ??:0
@ 0x7fc25046cc0a _ZNSt17_Function_handlerIFvvEZN4kudu7tserver15TSTabletManager15StartTabletCopyEPKNS1_9consensus24StartTabletCopyRequestPBESt8functionIFvRKNS1_6StatusENS2_24TabletServerErrorPB_CodeEEEEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x564c1473375c std::function<>::operator()() at ??:0
@ 0x7fc24d826e06 kudu::ThreadPool::DispatchThread() at ??:0
@ 0x7fc24d8276ef _ZZN4kudu10ThreadPool12CreateThreadEvENKUlvE_clEv at ??:0
@ 0x7fc24d82903b _ZNSt17_Function_handlerIFvvEZN4kudu10ThreadPool12CreateThreadEvEUlvE_E9_M_invokeERKSt9_Any_data at ??:0
@ 0x564c1473375c std::function<>::operator()() at ??:0
@ 0x7fc24d816807 kudu::Thread::SuperviseThread() at ??:0
@ 0x7fc24e0546db start_thread at ??:0
@ 0x7fc24b58c71f clone at ??:0
W20240501 08:01:48.065364 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:01:48.310189 1319 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet d968c4eddef4443a8a75238946e4c2bb after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.976s (SENT)
W20240501 08:02:47.357007 1319 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385) has failed: Timed out: Write RPC to 127.0.99.6:39385 timed out after 59.994s (SENT)
W20240501 08:02:47.357127 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 2c9e8ce22edd4d479d92fef37147933e after 3 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.994s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 15ef004a6c104b9fa5f712096b4fcbdd of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 175 | Yes
B | [config not available] | | |
C | A* B C | 2 | 175 | Yes
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet bd276a242f11451d85f8d3b3b6a224a9 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 179 | Yes
B | [config not available] | | |
C | A* B C | 2 | 179 | Yes
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet d2be42fb0d6143b79ea8e7f4bba4f4eb of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 181 | Yes
B | [config not available] | | |
C | A* B C | 2 | 181 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 61e49eb5434f451892cd1ea845ca8af3 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 3 | 185 | Yes
B | [config not available] | | |
C | A* B C | 3 | 185 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet c712ac72cb614d089d660062a92942c1 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 1 | 184 | Yes
B | [config not available] | | |
C | A* B C | 1 | 184 | Yes
Tablet 416c2905133b47cd9eb056c890af2e53 of table 'RemoveReplaceInCycle' is under-replicated: 1 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 6ee13bbcbfc142cfac96ca0721cd8081
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C | 3 | 181 | Yes
B | [config not available] | | |
C | A* B C | 3 | 181 | Yes
D | [config not available] | | |
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 10 | 0 | 6 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 18
First Quartile | 21
Median | 23
Third Quartile | 26
Maximum | 26
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 137
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:02:48.358299 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:02:49.364771 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:02:50.381208 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:02:51.398821 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:02:52.420545 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:02:53.424564 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:02:54.455143 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:02:55.455286 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:02:56.464042 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:02:57.481598 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:02:58.482050 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:02:59.484522 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:00.534274 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:01.566293 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:02.577383 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:03.582501 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:04.658450 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:05.660861 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:06.679531 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:07.740134 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:08.754035 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:09.821242 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:10.849759 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:11.938417 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:12.973606 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:14.020061 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:15.030997 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:16.070916 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:17.071064 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:18.248258 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:19.260279 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:20.260802 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:21.290550 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:22.337947 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:23.401063 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:24.477979 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:25.574517 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:26.680430 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:27.802726 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:28.828011 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:29.833345 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:30.851060 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:31.868494 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:33.064795 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:34.120044 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:35.201104 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:36.270674 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:37.383030 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:38.638989 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:39.660856 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:40.736691 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:41.751963 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:42.874440 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:43.875522 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:44.881373 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:45.900578 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:46.928082 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:47.650507 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet ffb1398cd75340bf8c8f8944db756435 after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.990s (SENT)
W20240501 08:03:48.006598 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:49.027192 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:50.028302 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:51.048990 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:52.058564 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:53.065830 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:54.074548 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:55.075285 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:03:56.099290 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:57.114625 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:03:58.114679 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:03:59.127954 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:00.193984 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:01.281275 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:02.282214 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:03.314471 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:04.382885 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:05.482645 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:06.489336 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:07.534961 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:08.614574 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:09.645617 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:10.706099 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:11.782758 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:12.880772 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:13.906596 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:14.937157 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:15.999439 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:17.070789 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:18.085522 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:19.274703 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:20.278611 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:21.322415 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:22.343272 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:23.401417 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:24.489478 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:25.558357 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:26.569710 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:27.672260 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:28.695696 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:29.782809 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:30.837970 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:31.857357 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:32.906755 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:33.928450 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:35.146600 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:36.263885 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:37.320688 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:38.328284 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:39.419915 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:40.534534 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:41.540019 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:42.545598 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:43.804266 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:44.810869 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:45.826474 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:46.854187 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:47.890480 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:47.999902 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet d968c4eddef4443a8a75238946e4c2bb after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.993s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:04:48.895059 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:49.895349 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:50.904467 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:51.911628 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:52.916913 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:53.968823 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:54.970898 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:55.995205 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:04:57.017385 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:04:58.022938 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:04:59.053725 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:00.094823 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:01.101974 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:02.139626 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:03.145865 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:04.178983 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:05.245008 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:06.264021 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:07.318822 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:08.320469 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:09.321544 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:10.324865 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:11.378574 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:12.459342 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:13.560971 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:14.583074 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:15.607518 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:16.676173 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:17.741173 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:18.751418 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:19.954627 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:20.960402 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:21.981732 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:23.020807 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:24.079612 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:25.155310 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:26.244457 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:27.344206 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:28.344913 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:29.356923 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:30.472687 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:31.505239 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:32.517298 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:33.570849 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:34.620859 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:35.831413 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:36.915239 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:38.003857 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:39.112255 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:40.231899 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:41.243280 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:42.504032 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:43.514976 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:44.524528 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:45.531235 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:46.538610 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:47.538877 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:48.558956 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:48.597184 1318 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet f8180cedc3f14f9f95855877ba5f7cb0 after 1 attempt(s): Failed to write to server: a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): Write RPC to 127.0.99.1:38487 timed out after 59.998s (SENT)
W20240501 08:05:49.564074 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:50.571691 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:51.576997 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:52.580222 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:53.606312 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:05:54.621546 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:55.623843 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:56.657686 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:57.687772 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:05:58.708022 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:05:59.724203 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:00.740859 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:01.773556 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:02.799633 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:03.813927 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:04.814954 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:05.868528 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:06.899684 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:07.906533 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:08.945652 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:09.979413 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:11.002398 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:12.004890 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:13.052897 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:14.057618 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:15.062923 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:16.089123 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:17.098794 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:18.104097 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:19.130744 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:20.200474 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:21.212986 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:22.237645 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:23.276935 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:24.333582 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:25.346421 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:26.359282 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:27.361292 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:28.372978 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:29.427511 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:30.511088 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:31.585147 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:32.607401 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:33.653090 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:34.703073 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:35.705819 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:36.744796 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:37.775275 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:38.882022 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:39.893078 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:40.894816 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:41.924031 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:42.970019 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:43.995184 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:45.001022 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:46.004465 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:47.020009 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:48.043150 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:48.929339 1319 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet 3985d26e8567489986ec34b097d894ae after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.995s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:06:49.551343 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:06:50.572822 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:51.590718 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:52.600149 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:53.607530 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:54.620033 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:55.669461 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:56.700861 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:57.746230 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:06:58.753671 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:06:59.774709 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:00.796492 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:01.809165 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:02.918879 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:03.932876 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:04.971576 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:05.979997 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:07.083853 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:08.086177 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:09.132679 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:10.133610 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:11.228521 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:12.278815 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:13.292503 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:14.292694 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:15.358755 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:16.364431 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:17.369642 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:18.392503 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:19.417426 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:20.418054 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:21.421885 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:22.437551 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:23.450603 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:24.499161 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:25.511286 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:26.594419 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:27.695643 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:28.813633 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:29.819608 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:30.844553 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:31.950045 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:33.179028 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:34.181128 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:35.221802 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:36.288668 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:37.356580 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:38.649145 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:39.710230 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:40.820448 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:41.825593 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:42.840963 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:43.913336 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:45.072582 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:46.074999 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:47.092092 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:48.111865 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:49.144806 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:49.545287 1319 batcher.cc:441] Timed out: Failed to write batch of 3 ops to tablet 1d2de061358a43cdbd1b96e945dd8153 after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.997s (SENT)
W20240501 08:07:50.148342 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:51.158833 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:52.160028 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:53.160773 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:54.174000 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:55.216104 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:07:56.219980 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:07:57.229574 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:58.231850 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:07:59.258759 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:00.283212 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:01.305851 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:02.309434 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:03.309723 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:04.309749 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:05.342579 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:06.377758 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:07.397774 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:08.535054 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:09.539646 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:10.541121 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:11.550045 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:12.598829 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:13.677722 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:14.784735 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:15.794121 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:16.832584 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:17.878237 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:18.957692 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:19.968518 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:20.972398 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:22.045707 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:23.069855 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:24.240842 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:25.297863 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:26.374876 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:27.461709 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:28.560534 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:29.682993 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:30.815073 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:31.838706 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:32.842746 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:33.864487 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:34.924153 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:36.127395 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:37.190827 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:38.287899 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:39.366834 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:40.488476 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:41.762286 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:42.769744 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:43.875991 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:45.030699 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:46.034449 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:47.046168 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:48.066824 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:49.095121 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:49.886875 1319 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet ffb1398cd75340bf8c8f8944db756435 after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.989s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.1:38487, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:08:50.447261 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:51.447680 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:52.453649 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:53.465391 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:54.472980 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:08:55.501663 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:56.520977 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:57.550853 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:08:58.581709 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:08:59.721464 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:00.752593 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:01.870721 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:02.949277 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:03.978941 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:04.982164 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:06.021438 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:07.039955 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:08.052386 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:09.099738 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:10.276280 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:11.278913 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:12.316229 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:13.372539 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:14.456741 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:15.562433 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:16.699390 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:17.710219 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:18.797770 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:19.851751 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:20.995759 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:22.066509 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:23.256783 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:24.285274 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:25.327689 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:26.394089 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:27.474525 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:28.575356 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:29.689486 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:30.817978 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:31.825070 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:32.828281 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:34.016471 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:35.026971 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:36.093663 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:37.114828 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:38.231986 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:39.497354 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:40.538661 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:41.548293 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:42.660653 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:43.756171 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:44.761999 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:45.791496 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:46.795092 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:47.807101 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:48.831789 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:49.862830 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:50.435274 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 1ab58726b4904b95a2c6d2a5daca65b8 after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.998s (SENT)
W20240501 08:09:50.866272 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:50.866272 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:51.878302 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:52.881248 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:09:53.894345 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:54.897379 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:09:55.941511 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:56.941985 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:57.950902 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:09:58.961055 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:00.003420 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:01.006270 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:02.007248 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:03.081754 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:04.122570 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:05.124353 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:06.165469 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:07.174391 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:08.255790 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:09.297641 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:10.366346 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:11.367893 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:12.391932 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:13.444131 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:14.519376 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:15.528213 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:16.528322 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:17.570833 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:18.608841 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:19.690971 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:20.777881 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:21.795325 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:22.800297 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:23.820696 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:24.824992 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:25.876695 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:26.944664 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:28.025876 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:29.125501 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:30.135298 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:31.266957 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:32.274752 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:33.282449 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:34.283535 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:35.339152 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:36.356513 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:37.449059 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:38.511844 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:39.630817 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:40.678854 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:41.685689 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:42.798470 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:43.889864 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:44.914001 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:45.914561 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:46.936767 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:47.949193 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:48.967425 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:49.996471 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:50.732976 1318 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet ac8116c4d57d4891b118fb098266a104 after 1 attempt(s): Failed to write to server: a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): Write RPC to 127.0.99.1:38487 timed out after 59.994s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:10:51.286538 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:52.289892 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:53.293762 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:54.296149 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:10:55.304821 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:10:56.318034 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:57.337852 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:58.364166 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:10:59.374290 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:00.401962 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:01.422472 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:02.437809 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:03.478580 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:04.575877 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:05.595816 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:06.607978 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:07.675256 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:08.782300 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:09.923822 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:10.938087 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:11.940793 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:12.947738 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:13.989749 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:15.069103 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:16.165475 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:17.186283 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:18.208150 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:19.264678 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:20.336556 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:21.342809 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:22.345177 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:23.350564 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:24.362138 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:25.373661 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:26.411291 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:27.478921 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:28.558602 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:29.658746 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:30.669708 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:31.670486 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:32.814337 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:33.927424 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:34.968797 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:35.994987 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:37.062922 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:38.155474 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:39.348963 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:40.462489 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:41.550058 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:42.567490 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:43.659705 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:44.777429 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:45.791051 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:46.813908 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:47.817962 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:48.832383 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:49.855724 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:50.884022 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:51.285192 1319 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet 1d2de061358a43cdbd1b96e945dd8153 after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.997s (SENT)
W20240501 08:11:51.889312 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:11:52.906916 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:53.920114 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:54.924274 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:55.938496 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:11:56.940625 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:57.943955 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:58.948953 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:11:59.959240 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:00.985275 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:01.992455 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:03.004207 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:04.009356 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:05.055399 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:06.062651 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:07.106396 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:08.180879 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:09.194337 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:10.234860 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:11.235149 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:12.236660 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:13.237980 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:14.308070 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:15.366169 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:16.469461 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:17.504528 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:18.516537 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:19.532861 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:20.644222 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:21.665681 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:22.706737 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:23.707990 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:24.715608 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:25.768800 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:26.771124 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:27.791685 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:28.895180 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:29.980381 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:31.096743 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:32.102098 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:33.107515 1318 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:34.250046 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:35.292634 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:36.318518 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:37.325985 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:38.440277 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:39.468672 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:40.606375 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:41.639247 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:42.681823 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:12:43.796295 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:44.825367 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:45.831249 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:46.847383 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:47.912374 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:48.920677 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:49.938659 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:50.964941 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:51.583583 1318 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet ff8fbf4522a44e8887754461665a4349 after 1 attempt(s): Failed to write to server: a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): Write RPC to 127.0.99.1:38487 timed out after 59.992s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:12:52.216202 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:53.224188 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:54.264360 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:55.330996 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:56.353926 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:12:57.384481 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:58.403904 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:12:59.411912 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:00.413194 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:01.455381 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:02.483855 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:03.601553 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:04.633239 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:05.643715 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:06.646013 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:07.653550 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:08.703842 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:09.704265 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:10.822587 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:11.826653 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:12.827307 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:13.834950 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:14.887367 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:15.909670 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:17.061183 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:18.104840 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:19.106397 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:20.116161 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:21.236565 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:22.237250 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:23.246968 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:24.253123 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:25.269783 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:26.275754 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:27.315801 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:28.368021 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:29.369836 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:30.565888 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:31.579645 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:32.600653 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:33.662700 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:34.827980 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:35.896821 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:36.897560 1317 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:37.993912 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:39.021728 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:40.022076 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:41.191226 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:42.208413 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:43.276355 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:44.370726 1317 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:45.385596 1317 meta_cache.cc:302] tablet 8a6f3866eab04807933d16a58e274345: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:46.402539 1318 meta_cache.cc:302] tablet ba20ceb60c1541749c404f5d98d7cd5e: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:47.515940 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:48.519188 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:49.530539 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:50.554114 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:51.579209 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:52.211671 1319 batcher.cc:441] Timed out: Failed to write batch of 2 ops to tablet d968c4eddef4443a8a75238946e4c2bb after 1 attempt(s): Failed to write to server: 883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): Write RPC to 127.0.99.6:39385 timed out after 59.992s (SENT)
W20240501 08:13:52.582156 1318 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:53.593087 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:54.594238 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:13:55.615728 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:56.636210 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:13:57.658136 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:58.658536 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:13:59.672349 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:00.682677 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:01.734926 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:02.740504 1318 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:03.744655 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:04.817939 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:05.837114 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:06.850695 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:07.868994 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:08.887521 1318 meta_cache.cc:302] tablet b1266d9b2ba14491a43470c4c3a672d3: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:09.973135 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:10.994351 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:11.998832 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:13.046489 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:14.122953 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:15.204879 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:16.263042 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:17.372061 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:18.400513 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:19.416895 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:20.439211 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:21.551432 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:22.605845 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:23.765686 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:24.768615 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:25.791227 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:26.837661 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:27.889107 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:28.956321 1318 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:30.051753 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:31.148864 1317 meta_cache.cc:302] tablet 3428d0607c4e4abd9ce794ddfa180915: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:32.275704 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:33.410748 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:34.426183 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:35.434418 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:36.471496 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:37.508747 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:38.717235 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:39.725508 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:40.725602 1317 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:41.812155 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:42.867641 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:43.871711 1318 meta_cache.cc:302] tablet fea0a0c497e044e9b383ebaf6f3a675a: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:45.005941 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:46.014900 1318 meta_cache.cc:302] tablet 2c9e8ce22edd4d479d92fef37147933e: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:47.065017 1317 meta_cache.cc:302] tablet c540bda6e2aa4d7dbb16078aa856b114: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:48.292496 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:49.300205 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:50.317756 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:51.338860 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:52.371821 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:52.497339 1318 batcher.cc:441] Timed out: Failed to write batch of 1 ops to tablet be6a13abc3b143dd97f3edfa0dd50005 after 1 attempt(s): Failed to write to server: a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): Write RPC to 127.0.99.1:38487 timed out after 59.990s (SENT)
Master Summary
UUID | Address | Status
----------------------------------+-------------------+---------
a0a523a69ec7463bafbcd5cbbe13f81e | 127.0.99.62:38485 | HEALTHY
Unusual flags for Master:
Flag | Value | Tags | Master
------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+-------------------------
ipki_ca_key_size | 768 | experimental | all 1 server(s) checked
ipki_server_key_size | 768 | experimental | all 1 server(s) checked
never_fsync | true | unsafe,advanced | all 1 server(s) checked
openssl_security_level_override | 0 | unsafe,hidden | all 1 server(s) checked
raft_prepare_replacement_before_eviction | true | experimental,advanced | all 1 server(s) checked
rpc_reuseport | true | experimental | all 1 server(s) checked
rpc_server_allow_ephemeral_ports | true | unsafe | all 1 server(s) checked
server_dump_info_format | pb | hidden | all 1 server(s) checked
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/master-0/data/info.pb | hidden | all 1 server(s) checked
tsk_num_rsa_bits | 512 | experimental | all 1 server(s) checked
Flags of checked categories for Master:
Flag | Value | Master
---------------------+-------------------+-------------------------
builtin_ntp_servers | 127.0.99.20:36269 | all 1 server(s) checked
time_source | builtin | all 1 server(s) checked
Tablet Server Summary
UUID | Address | Status | Location | Tablet Leaders | Active Scanners
----------------------------------+------------------+-------------+----------+----------------+-----------------
6ee13bbcbfc142cfac96ca0721cd8081 | 127.0.99.3:37779 | HEALTHY | <none> | 6 | 0
883d17aecdcb49d886fb9b5af1b1b571 | 127.0.99.6:39385 | HEALTHY | <none> | 10 | 0
a80601f7114d41e2abd9b423d514e048 | 127.0.99.1:38487 | HEALTHY | <none> | 13 | 0
3b978884440b466f8682dbad60b636d8 | 127.0.99.5:34987 | UNAVAILABLE | <none> | n/a | n/a
55f6008aeaae4a8d9d2c0dc6405be801 | 127.0.99.4:43085 | UNAVAILABLE | <none> | n/a | n/a
5b14472270734421a5284e2b5e7ee17a | 127.0.99.2:46683 | UNAVAILABLE | <none> | n/a | n/a
Tablet Server Location Summary
Location | Count
----------+---------
<none> | 6
Error from 127.0.99.5:34987: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.4:43085: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111) (UNAVAILABLE)
Error from 127.0.99.2:46683: Network error: could not get status from server: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111) (UNAVAILABLE)
Unusual flags for Tablet Server:
Flag | Value | Tags | Tablet Server
------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-----------------------+------------------------------------------------------
ipki_server_key_size | 768 | experimental | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.1 | experimental | 127.0.99.1:38487
local_ip_for_outbound_sockets | 127.0.99.3 | experimental | 127.0.99.3:37779
local_ip_for_outbound_sockets | 127.0.99.6 | experimental | 127.0.99.6:39385
never_fsync | true | unsafe,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
openssl_security_level_override | 0 | unsafe,hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
raft_prepare_replacement_before_eviction | true | experimental,advanced | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
rpc_server_allow_ephemeral_ports | true | unsafe | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_format | pb | hidden | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-0/data/info.pb | hidden | 127.0.99.1:38487
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-2/data/info.pb | hidden | 127.0.99.3:37779
server_dump_info_path | /tmp/dist-test-taskvJpsOF/test-tmp/raft_consensus_stress-itest.0.RaftConsensusStressITest.RemoveReplaceInCycle.1714550433713564-396-0/raft_consensus-itest-cluster/ts-5/data/info.pb | hidden | 127.0.99.6:39385
tablet_copy_early_session_timeout_prob | 0.25 | unsafe,runtime | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Flags of checked categories for Tablet Server:
Flag | Value | Tablet Server
---------------------+-------------------+------------------------------------------------------
builtin_ntp_servers | 127.0.99.20:36269 | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
time_source | builtin | 127.0.99.3:37779, 127.0.99.6:39385, 127.0.99.1:38487
Version Summary
Version | Servers
-----------------+-----------------------------------------------------------------------------------------------------
1.18.0-SNAPSHOT | master@127.0.99.62:38485, tserver@127.0.99.3:37779, tserver@127.0.99.6:39385, and 1 other server(s)
Tablet Summary
Tablet f8180cedc3f14f9f95855877ba5f7cb0 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 3b978884440b466f8682dbad60b636d8
C = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C | | | Yes
A | A* B C | 2 | 129 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet d968c4eddef4443a8a75238946e4c2bb of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
6ee13bbcbfc142cfac96ca0721cd8081 (127.0.99.3:37779): RUNNING [NONVOTER]
All reported replicas are:
A = 5b14472270734421a5284e2b5e7ee17a
B = 55f6008aeaae4a8d9d2c0dc6405be801
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 6ee13bbcbfc142cfac96ca0721cd8081
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D | 1 | 62 | No
D | A B C* D | 1 | 62 | No
Tablet 19ab4ec652df4e178d2c3f771c899580 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D~ | 2 | 123 | Yes
D | [config not available] | | |
Tablet 2144cdf57aa04828901e481213cddd02 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 118 | Yes
Tablet be6a13abc3b143dd97f3edfa0dd50005 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 67 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 3428d0607c4e4abd9ce794ddfa180915 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 120 | Yes
Tablet ff8fbf4522a44e8887754461665a4349 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 2 | 66 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet ffb1398cd75340bf8c8f8944db756435 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | Yes
D | [config not available] | | |
Tablet 1ab58726b4904b95a2c6d2a5daca65b8 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 121 | No
Tablet ac8116c4d57d4891b118fb098266a104 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 72 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 1d2de061358a43cdbd1b96e945dd8153 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* | 1 | 112 | Yes
Tablet fea0a0c497e044e9b383ebaf6f3a675a of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 1 | 124 | Yes
Tablet 3985d26e8567489986ec34b097d894ae of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 125 | No
Tablet 4241e2bd94bf4cb0877b7fc9c7d20e4c of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 3 | 120 | Yes
D | [config not available] | | |
Tablet 8b471bd5fab04ca2b38fea49facb6a62 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING [LEADER]
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A* B C D~ | | | Yes
A | A* B C D | 1 | 69 | No
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 8a6f3866eab04807933d16a58e274345 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 116 | Yes
Tablet ba20ceb60c1541749c404f5d98d7cd5e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 63 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet 2c9e8ce22edd4d479d92fef37147933e of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 2 | 66 | No
D | [config not available] | | |
Tablet b1266d9b2ba14491a43470c4c3a672d3 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 58 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet b27ed99ebb484a798006727a3fdba1fa of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [NONVOTER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C D | 1 | 56 | No
D | [config not available] | | |
Tablet 7cf873316527452990569bfcc6bfe238 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | A B* C | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
Tablet 52f43c86fc584a6b87ddbf2cfa150399 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
a80601f7114d41e2abd9b423d514e048 (127.0.99.1:38487): RUNNING
5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683): TS unavailable [LEADER]
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [NONVOTER]
All reported replicas are:
A = a80601f7114d41e2abd9b423d514e048
B = 5b14472270734421a5284e2b5e7ee17a
C = 55f6008aeaae4a8d9d2c0dc6405be801
D = 3b978884440b466f8682dbad60b636d8
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C D~ | | | Yes
A | A B* C D~ | 1 | 72 | Yes
B | [config not available] | | |
C | [config not available] | | |
D | [config not available] | | |
Tablet c540bda6e2aa4d7dbb16078aa856b114 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable [LEADER]
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B* C | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B* C | 2 | 122 | Yes
Tablet 3d3d4a35a725452da7e1542ce5c4e8b9 of table 'RemoveReplaceInCycle' is unavailable: 2 replica(s) not RUNNING
55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085): TS unavailable
3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987): TS unavailable
883d17aecdcb49d886fb9b5af1b1b571 (127.0.99.6:39385): RUNNING [LEADER]
All reported replicas are:
A = 55f6008aeaae4a8d9d2c0dc6405be801
B = 3b978884440b466f8682dbad60b636d8
C = 883d17aecdcb49d886fb9b5af1b1b571
D = 5b14472270734421a5284e2b5e7ee17a
The consensus matrix is:
Config source | Replicas | Current term | Config index | Committed?
---------------+------------------------+--------------+--------------+------------
master | A B C* | | | Yes
A | [config not available] | | |
B | [config not available] | | |
C | A B C* D~ | 2 | 119 | No
The cluster doesn't have any matching system tables
Summary by table
Name | RF | Status | Total Tablets | Healthy | Recovering | Under-replicated | Unavailable
----------------------+----+-------------+---------------+---------+------------+------------------+-------------
TestTable | 3 | HEALTHY | 1 | 1 | 0 | 0 | 0
RemoveReplaceInCycle | 3 | UNAVAILABLE | 40 | 16 | 0 | 0 | 24
Tablet Replica Count Summary
Statistic | Replica Count
----------------+---------------
Minimum | 15
First Quartile | 18
Median | 23
Third Quartile | 26
Maximum | 32
Total Count Summary
| Total Count
----------------+-------------
Masters | 1
Tablet Servers | 6
Tables | 2
Tablets | 41
Replicas | 136
==================
Warnings:
==================
Some masters have unsafe, experimental, or hidden flags set
Some tablet servers have unsafe, experimental, or hidden flags set
tserver unusual flags check error: 3 of 6 tservers were not available to retrieve unusual flags
tserver diverged flags check error: 3 of 6 tservers were not available to retrieve time_source category flags
==================
Errors:
==================
Network error: error fetching info from tablet servers: failed to gather info for all tablet servers: 3 of 6 had errors
Corruption: table consistency check error: 1 out of 2 table(s) are not healthy
FAILED
W20240501 08:14:53.380792 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:54.386703 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:55.391405 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:56.438973 1318 meta_cache.cc:302] tablet 7cf873316527452990569bfcc6bfe238: replica 5b14472270734421a5284e2b5e7ee17a (127.0.99.2:46683) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.2:46683: connect: Connection refused (error 111)
W20240501 08:14:57.442871 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:14:58.443328 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:14:59.470887 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:15:00.478107 1317 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:15:01.535368 1317 meta_cache.cc:302] tablet b27ed99ebb484a798006727a3fdba1fa: replica 3b978884440b466f8682dbad60b636d8 (127.0.99.5:34987) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.5:34987: connect: Connection refused (error 111)
W20240501 08:15:02.549384 1318 meta_cache.cc:302] tablet 19ab4ec652df4e178d2c3f771c899580: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
W20240501 08:15:03.591259 1318 meta_cache.cc:302] tablet 52f43c86fc584a6b87ddbf2cfa150399: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
************************ BEGIN STACKS **************************
[New LWP 399]
[New LWP 400]
[New LWP 401]
[New LWP 402]
[New LWP 403]
[New LWP 404]
[New LWP 405]
[New LWP 1285]
[New LWP 1286]
[New LWP 1287]
[New LWP 1288]
[New LWP 1317]
[New LWP 1318]
[New LWP 1319]
[New LWP 1320]
[New LWP 1856]
[New LWP 1857]
[New LWP 11531]
[New LWP 11675]
[New LWP 11689]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
Id Target Id Frame
* 1 Thread 0x7f9b0fbb0380 (LWP 396) "raft_consensus_" 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
2 Thread 0x7f9b0db8b700 (LWP 399) "raft_consensus_" 0x00007f9b1c26a32a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
3 Thread 0x7f9b0c1ae700 (LWP 400) "kernel-watcher-" 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
4 Thread 0x7f9b0b9ad700 (LWP 401) "rpc reactor-401" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
5 Thread 0x7f9b0b1ac700 (LWP 402) "rpc reactor-402" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
6 Thread 0x7f9b0a9ab700 (LWP 403) "rpc reactor-403" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
7 Thread 0x7f9b0a1aa700 (LWP 404) "rpc reactor-404" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
8 Thread 0x7f9b099a9700 (LWP 405) "rpc reactor-405" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
9 Thread 0x7f9b091a8700 (LWP 1285) "rpc reactor-128" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
10 Thread 0x7f9b087a0700 (LWP 1286) "rpc reactor-128" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
11 Thread 0x7f9b07f9f700 (LWP 1287) "rpc reactor-128" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
12 Thread 0x7f9b0779e700 (LWP 1288) "rpc reactor-128" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
13 Thread 0x7f9b05f9b700 (LWP 1317) "rpc reactor-131" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
14 Thread 0x7f9b0579a700 (LWP 1318) "rpc reactor-131" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
15 Thread 0x7f9b04f99700 (LWP 1319) "rpc reactor-131" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
16 Thread 0x7f9b04798700 (LWP 1320) "rpc reactor-132" 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
17 Thread 0x7f9b03f97700 (LWP 1856) "raft_consensus_" 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
18 Thread 0x7f9b02794700 (LWP 1857) "raft_consensus_" 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
19 Thread 0x7f9b02f95700 (LWP 11531) "raft_consensus_" 0x00007f9b1c265ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
20 Thread 0x7f9b03796700 (LWP 11675) "client-negotiat" 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
21 Thread 0x7f9b06f9d700 (LWP 11689) "client-negotiat" 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
Thread 21 (Thread 0x7f9b06f9d700 (LWP 11689)):
#0 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1bd249d4 in kudu::ConditionVariable::WaitFor (this=0x7f9b06f9c690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007f9b1be728d1 in kudu::ThreadPool::DispatchThread (this=0x55d9227ecf00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007f9b1be736ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55d9229cbf58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007f9b1be7503b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d9229cbf58) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d9229cbf00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 20 (Thread 0x7f9b03796700 (LWP 11675)):
#0 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1bd249d4 in kudu::ConditionVariable::WaitFor (this=0x7f9b03795690, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:121
#2 0x00007f9b1be728d1 in kudu::ThreadPool::DispatchThread (this=0x55d92293af00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:720
#3 0x00007f9b1be736ef in kudu::ThreadPool::<lambda()>::operator()(void) const (__closure=0x55d9229cac58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/threadpool.cc:848
#4 0x00007f9b1be7503b in std::_Function_handler<void(), kudu::ThreadPool::CreateThread()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#5 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d9229cac58) at /usr/include/c++/7/bits/std_function.h:706
#6 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d9229cac00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#7 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#8 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 19 (Thread 0x7f9b02f95700 (LWP 11531)):
#0 0x00007f9b1c265ad3 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1bd24665 in kudu::ConditionVariable::Wait (this=0x55d922be8990) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:55
#2 0x00007f9b1caae4c1 in kudu::client::KuduSession::Data::Flush (this=0x55d922be8900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/session-internal.cc:257
#3 0x00007f9b1c9b8638 in kudu::client::KuduSession::Flush (this=0x55d9229401e0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/client/client.cc:1524
#4 0x00007f9b1d2df7e3 in kudu::TestWorkload::WriteThread (this=0x7ffe2af0cde0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/test_workload.cc:204
#5 0x00007f9b1d2e9b8f in std::__invoke_impl<void, void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__f=@0x55d922d3e8f0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7f9b1d2ded88 <kudu::TestWorkload::WriteThread()>, __t=@0x55d922d3e8e8: 0x7ffe2af0cde0) at /usr/include/c++/7/bits/invoke.h:73
#6 0x00007f9b1d2e9436 in std::__invoke<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> (__fn=@0x55d922d3e8f0: (void (kudu::TestWorkload::*)(kudu::TestWorkload * const)) 0x7f9b1d2ded88 <kudu::TestWorkload::WriteThread()>) at /usr/include/c++/7/bits/invoke.h:95
#7 0x00007f9b1d2eab07 in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::_M_invoke<0ul, 1ul> (this=0x55d922d3e8e8) at /usr/include/c++/7/thread:234
#8 0x00007f9b1d2eaa6d in std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> >::operator() (this=0x55d922d3e8e8) at /usr/include/c++/7/thread:243
#9 0x00007f9b1d2ea9e2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (kudu::TestWorkload::*)(), kudu::TestWorkload*> > >::_M_run (this=0x55d922d3e8e0) at /usr/include/c++/7/thread:186
#10 0x00007f9b1ae2d6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#11 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#12 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 18 (Thread 0x7f9b02794700 (LWP 1857)):
#0 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1b3ec282 in base::SleepForNanoseconds (nanoseconds=1000000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f9b1be02a69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055d920dc79cc in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55d92295cbd8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:215
#4 0x000055d920dca299 in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055d920dc9ef9 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055d920dcad36 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55d92295cbd8) at /usr/include/c++/7/thread:234
#7 0x000055d920dcace3 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55d92295cbd8) at /usr/include/c++/7/thread:243
#8 0x000055d920dcaca2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55d92295cbd0) at /usr/include/c++/7/thread:186
#9 0x00007f9b1ae2d6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 17 (Thread 0x7f9b03f97700 (LWP 1856)):
#0 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1b3ec282 in base::SleepForNanoseconds (nanoseconds=250000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f9b1be02a69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055d920dc752c in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()>::operator()(void) const (__closure=0x55d92295c668) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:196
#4 0x000055d920dca12f in std::__invoke_impl<void, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(std::__invoke_other, kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#5 0x000055d920dc9df6 in std::__invoke<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> >(kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#6 0x000055d920dcad62 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55d92295c668) at /usr/include/c++/7/thread:234
#7 0x000055d920dcad07 in std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > >::operator()(void) (this=0x55d92295c668) at /usr/include/c++/7/thread:243
#8 0x000055d920dcacc2 in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody()::<lambda()> > > >::_M_run(void) (this=0x55d92295c660) at /usr/include/c++/7/thread:186
#9 0x00007f9b1ae2d6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#10 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#11 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 16 (Thread 0x7f9b04798700 (LWP 1320)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228e1600, timeout=0.099261227164788579) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228e1600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922841560, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922841558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d9229d6658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d9229d6658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d9229d6600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 15 (Thread 0x7f9b04f99700 (LWP 1319)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228e1180, timeout=0.099435932164851693) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228e1180, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d9228413a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922841398) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d9229d6558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d9229d6558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d9229d6500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 14 (Thread 0x7f9b0579a700 (LWP 1318)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228e0d00, timeout=0.079687767999985226) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228e0d00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d9228411e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d9228411d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274ce58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274ce58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274ce00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 13 (Thread 0x7f9b05f9b700 (LWP 1317)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228e0880, timeout=0.026381920000176251) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228e0880, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922841020, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922841018) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274d658) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274d658) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274d600) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 12 (Thread 0x7f9b0779e700 (LWP 1288)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228e0400, timeout=0.099885299164725438) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228e0400, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840e60, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840e58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274c358) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274c358) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274c300) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 11 (Thread 0x7f9b07f9f700 (LWP 1287)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228dff80, timeout=0.099600221164791947) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228dff80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840ca0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840c98) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274c958) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274c958) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274c900) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 10 (Thread 0x7f9b087a0700 (LWP 1286)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228dfb00, timeout=0.099601764164845008) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228dfb00, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840ae0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840ad8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274c558) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274c558) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274c500) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 9 (Thread 0x7f9b091a8700 (LWP 1285)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228df680, timeout=0.099735712164829238) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228df680, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840920, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840918) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274c758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274c758) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274c700) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 8 (Thread 0x7f9b099a9700 (LWP 405)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228ded80, timeout=0.099041221164725357) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228ded80, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840220, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840218) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274d458) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274d458) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274d400) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 7 (Thread 0x7f9b0a1aa700 (LWP 404)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228de900, timeout=0.099403696164699795) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228de900, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d9228403e0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d9228403d8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274d058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274d058) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274d000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 6 (Thread 0x7f9b0a9ab700 (LWP 403)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228de480, timeout=0.099111320164865901) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228de480, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840760, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840758) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274c858) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274c858) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274c800) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 5 (Thread 0x7f9b0b1ac700 (LWP 402)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9228de000, timeout=0.099050308164578382) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9228de000, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d922840060, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840058) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274d258) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274d258) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274d200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 4 (Thread 0x7f9b0b9ad700 (LWP 401)):
#0 0x00007f9b1a4eaa47 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6
#1 0x00007f9b15cedde0 in epoll_poll (loop=0x55d9227a1600, timeout=0.099325588164674627) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev_epoll.c:153
#2 0x00007f9b15cf149e in ev_run (loop=0x55d9227a1600, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/libev-4.20/ev.c:3521
#3 0x00007f9b1be5bba3 in ev::loop_ref::run (this=0x55d9228405a0, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/ev++.h:211
#4 0x00007f9b1867031d in kudu::rpc::ReactorThread::RunThread (this=0x55d922840598) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:510
#5 0x00007f9b1866cc49 in kudu::rpc::ReactorThread::<lambda()>::operator()(void) const (__closure=0x55d92274cc58) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/rpc/reactor.cc:197
#6 0x00007f9b18674934 in std::_Function_handler<void(), kudu::rpc::ReactorThread::Init()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274cc58) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274cc00) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 3 (Thread 0x7f9b0c1ae700 (LWP 400)):
#0 0x00007f9b1c265fb9 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1bd247eb in kudu::ConditionVariable::WaitUntil (this=0x55d9227352d8, until=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/condition_variable.cc:87
#2 0x00007f9b1cdfcfde in kudu::CountDownLatch::WaitUntil (this=0x55d9227352a0, when=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:87
#3 0x00007f9b1cdfd088 in kudu::CountDownLatch::WaitFor (this=0x55d9227352a0, delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/countdown_latch.h:97
#4 0x00007f9b1bdbc166 in kudu::KernelStackWatchdog::RunThread (this=0x55d922735200) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:134
#5 0x00007f9b1bdbb375 in kudu::KernelStackWatchdog::<lambda()>::operator()(void) const (__closure=0x55d92274d158) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/kernel_stack_watchdog.cc:71
#6 0x00007f9b1bdbcaca in std::_Function_handler<void(), kudu::KernelStackWatchdog::KernelStackWatchdog()::<lambda()> >::_M_invoke(const std::_Any_data &) (__functor=...) at /usr/include/c++/7/bits/std_function.h:316
#7 0x00007f9b1d0ef856 in std::function<void ()>::operator()() const (this=0x55d92274d158) at /usr/include/c++/7/bits/std_function.h:706
#8 0x00007f9b1be62807 in kudu::Thread::SuperviseThread (arg=0x55d92274d100) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/thread.cc:694
#9 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#10 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 2 (Thread 0x7f9b0db8b700 (LWP 399)):
#0 0x00007f9b1c26a32a in waitpid () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1be597b5 in kudu::Subprocess::DoWait (this=0x7f9b0db8a380, wait_status=0x0, mode=kudu::Subprocess::BLOCKING) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:817
#2 0x00007f9b1be56be8 in kudu::Subprocess::Wait (this=0x7f9b0db8a380, wait_status=0x0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/subprocess.cc:544
#3 0x00007f9b1bda9b7c in kudu::PstackWatcher::RunStackDump (argv=std::vector of length 12, capacity 16 = {...}) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:258
#4 0x00007f9b1bda96ab in kudu::PstackWatcher::RunGdbStackDump (pid=396, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:218
#5 0x00007f9b1bda8f09 in kudu::PstackWatcher::DumpPidStacks (pid=396, flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:169
#6 0x00007f9b1bda8e82 in kudu::PstackWatcher::DumpStacks (flags=0) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/pstack_watcher.cc:162
#7 0x00007f9b1d19a087 in kudu::<lambda()>::operator()(void) const (__closure=0x55d9227b39f8) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:60
#8 0x00007f9b1d19a888 in std::__invoke_impl<void, kudu::CreateAndStartTimeoutThread()::<lambda()> >(std::__invoke_other, kudu::<lambda()> &&) (__f=...) at /usr/include/c++/7/bits/invoke.h:60
#9 0x00007f9b1d19a5dc in std::__invoke<kudu::CreateAndStartTimeoutThread()::<lambda()> >(kudu::<lambda()> &&) (__fn=...) at /usr/include/c++/7/bits/invoke.h:95
#10 0x00007f9b1d19ae6e in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::_M_invoke<0>(std::_Index_tuple<0>) (this=0x55d9227b39f8) at /usr/include/c++/7/thread:234
#11 0x00007f9b1d19ae13 in std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > >::operator()(void) (this=0x55d9227b39f8) at /usr/include/c++/7/thread:243
#12 0x00007f9b1d19adce in std::thread::_State_impl<std::thread::_Invoker<std::tuple<kudu::CreateAndStartTimeoutThread()::<lambda()> > > >::_M_run(void) (this=0x55d9227b39f0) at /usr/include/c++/7/thread:186
#13 0x00007f9b1ae2d6df in ?? () from /usr/lib/x86_64-linux-gnu/libstdc++.so.6
#14 0x00007f9b1c25f6db in start_thread () from /lib/x86_64-linux-gnu/libpthread.so.0
#15 0x00007f9b1a4ea71f in clone () from /lib/x86_64-linux-gnu/libc.so.6
Thread 1 (Thread 0x7f9b0fbb0380 (LWP 396)):
#0 0x00007f9b1c269d50 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0
#1 0x00007f9b1b3ec282 in base::SleepForNanoseconds (nanoseconds=1000000) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/gutil/sysinfo.cc:96
#2 0x00007f9b1be02a69 in kudu::SleepFor (delta=...) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/monotime.cc:264
#3 0x000055d920dc88e1 in kudu::tserver::RaftConsensusStressITest_RemoveReplaceInCycle_Test::TestBody (this=0x55d9227a1a80) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/integration-tests/raft_consensus_stress-itest.cc:229
#4 0x00007f9b1c0050ed in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::Test, void> (location=0x7f9b1c00682b "the test body", method=<optimized out>, object=0x55d9227a1a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#5 testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void> (object=object@entry=0x55d9227a1a80, method=<optimized out>, location=location@entry=0x7f9b1c00682b "the test body") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#6 0x00007f9b1bff9bdb in testing::Test::Run (this=0x55d9227a1a80) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2674
#7 0x00007f9b1bff9d9d in testing::TestInfo::Run (this=0x55d922734240) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2853
#8 0x00007f9b1bffa377 in testing::TestSuite::Run (this=0x55d922734480) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:3012
#9 0x00007f9b1bffa77c in testing::internal::UnitTestImpl::RunAllTests (this=<optimized out>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5870
#10 0x00007f9b1c00560d in testing::internal::HandleSehExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (location=0x7f9b1c0083d8 "auxiliary test code (environments or event listeners)", method=<optimized out>, object=0x55d9227ec280) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2599
#11 testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool> (object=0x55d9227ec280, method=<optimized out>, location=location@entry=0x7f9b1c0083d8 "auxiliary test code (environments or event listeners)") at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:2635
#12 0x00007f9b1bff9e63 in testing::UnitTest::Run (this=0x7f9b1c2191e0 <testing::UnitTest::GetInstance()::instance>) at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/src/googletest-release-1.12.1/googletest/src/gtest.cc:5444
#13 0x00007f9b1d19b1de in RUN_ALL_TESTS () at /home/jenkins-slave/workspace/build_and_test_flaky/thirdparty/installed/uninstrumented/include/gtest/gtest.h:2293
#14 0x00007f9b1d19a53d in main (argc=1, argv=0x7ffe2af0d468) at /home/jenkins-slave/workspace/build_and_test_flaky/src/kudu/util/test_main.cc:109
#15 0x00007f9b1a3eabf7 in __libc_start_main () from /lib/x86_64-linux-gnu/libc.so.6
#16 0x000055d920dc6e4a in _start ()
W20240501 08:15:37.891012 1318 meta_cache.cc:302] tablet 2144cdf57aa04828901e481213cddd02: replica 55f6008aeaae4a8d9d2c0dc6405be801 (127.0.99.4:43085) has failed: Network error: Client connection negotiation failed: client connection to 127.0.99.4:43085: connect: Connection refused (error 111)
************************* END STACKS ***************************
F20240501 08:15:37.978291 399 test_main.cc:63] Maximum unit test time exceeded (870 sec)
*** Check failure stack trace: ***
*** Aborted at 1714551337 (unix time) try "date -d @1714551337" if you are using GNU date ***
PC: @ 0x0 (unknown)
*** SIGABRT (@0x3e80000018c) received by PID 396 (TID 0x7f9b0db8b700) from PID 396; stack trace: ***
@ 0x7f9b1c26a980 (unknown) at ??:0
@ 0x7f9b1a407fb7 gsignal at ??:0
@ 0x7f9b1a409921 abort at ??:0
@ 0x7f9b1b10cdcd google::LogMessage::Fail() at ??:0
@ 0x7f9b1b110b93 google::LogMessage::SendToLog() at ??:0
@ 0x7f9b1b10c7cc google::LogMessage::Flush() at ??:0
@ 0x7f9b1b10df59 google::LogMessageFatal::~LogMessageFatal() at ??:0
@ 0x7f9b1d19a1c4 _ZZN4kuduL27CreateAndStartTimeoutThreadEvENKUlvE_clEv at ??:0
@ 0x7f9b1d19a888 _ZSt13__invoke_implIvZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ at ??:0
@ 0x7f9b1d19a5dc _ZSt8__invokeIZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_JEENSt15__invoke_resultIT_JDpT0_EE4typeEOS3_DpOS4_ at ??:0
@ 0x7f9b1d19ae6e _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEE9_M_invokeIJLm0EEEEDTcl8__invokespcl10_S_declvalIXT_EEEEESt12_Index_tupleIJXspT_EEE at ??:0
@ 0x7f9b1d19ae13 _ZNSt6thread8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEclEv at ??:0
@ 0x7f9b1d19adce _ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJZN4kuduL27CreateAndStartTimeoutThreadEvEUlvE_EEEEE6_M_runEv at ??:0
@ 0x7f9b1ae2d6df (unknown) at ??:0
@ 0x7f9b1c25f6db start_thread at ??:0
@ 0x7f9b1a4ea71f clone at ??:0